feat(health-events-analyzer): recover derived conditions

commons/pkg/configmanager/loader.go

@@ -16,6 +16,7 @@ package configmanager
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/BurntSushi/toml"
 )
@@ -52,3 +53,24 @@ func LoadTOMLConfig[T any](path string, config *T) error {
 
 	return nil
 }
+
+// LoadTOMLConfigStrict loads TOML and rejects keys that are not represented in
+// the destination struct. Use it where a misspelled key must fail startup.
+func LoadTOMLConfigStrict[T any](path string, config *T) error {
+	metadata, err := toml.DecodeFile(path, config)
+	if err != nil {
+		return fmt.Errorf("failed to decode TOML file %s: %w", path, err)
+	}
+
+	undecoded := metadata.Undecoded()
+	if len(undecoded) == 0 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(undecoded))
+	for _, key := range undecoded {
+		keys = append(keys, key.String())
+	}
+
+	return fmt.Errorf("unknown TOML keys in %s: %s", path, strings.Join(keys, ", "))
+}

commons/pkg/configmanager/loader_test.go

@@ -17,6 +17,7 @@ package configmanager
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
 )
 
@@ -93,3 +94,50 @@ enabled = true
 		t.Fatal("expected error for invalid TOML syntax, got nil")
 	}
 }
+
+func TestLoadTOMLConfigStrictRejectsUnknownKeys(t *testing.T) {
+	t.Parallel()
+
+	configPath := filepath.Join(t.TempDir(), "config.toml")
+	contents := "name = \"test\"\nunknown_option = true\n"
+	if err := os.WriteFile(configPath, []byte(contents), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	var cfg testTOMLConfig
+	err := LoadTOMLConfigStrict(configPath, &cfg)
+	if err == nil || !strings.Contains(err.Error(), "unknown_option") {
+		t.Fatalf("LoadTOMLConfigStrict() error = %v", err)
+	}
+}
+
+func TestLoadTOMLConfigStrictAcceptsKnownKeys(t *testing.T) {
+	t.Parallel()
+
+	configPath := filepath.Join(t.TempDir(), "config.toml")
+	if err := os.WriteFile(configPath, []byte("name = \"test\"\nport = 8080\n"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	var cfg testTOMLConfig
+	if err := LoadTOMLConfigStrict(configPath, &cfg); err != nil {
+		t.Fatalf("LoadTOMLConfigStrict() error = %v", err)
+	}
+	if cfg.Name != "test" || cfg.Port != 8080 {
+		t.Fatalf("LoadTOMLConfigStrict() config = %#v", cfg)
+	}
+}
+
+func TestLoadTOMLConfigStrictRejectsInvalidTOML(t *testing.T) {
+	t.Parallel()
+
+	configPath := filepath.Join(t.TempDir(), "config.toml")
+	if err := os.WriteFile(configPath, []byte("name = [invalid"), 0o600); err != nil {
+		t.Fatal(err)
+	}
+
+	var cfg testTOMLConfig
+	if err := LoadTOMLConfigStrict(configPath, &cfg); err == nil {
+		t.Fatal("LoadTOMLConfigStrict() accepted invalid TOML")
+	}
+}

distros/kubernetes/nvsentinel/charts/mongodb-store/templates/_init-eval.tpl

@@ -81,6 +81,12 @@ db.$MONGODB_COLLECTION_NAME.createIndex({
   'healthevent.entitiesimpacted.entityvalue': 1,
   'healthevent.generatedtimestamp.seconds': 1
 });
+db.$MONGODB_COLLECTION_NAME.createIndex({
+  'healthevent.checkname': 1,
+  'healthevent.nodename': 1,
+  'createdAt': -1,
+  'healthevent.agent': 1
+});
 {{- if .Values.mongodb.tls.enabled }}
 // Create X.509 users (TLS only)
 var userExists = db.getSiblingDB('\$external').getUser('$MONGODB_APPLICATION_USER_DN');

distros/kubernetes/nvsentinel/templates/_external-mongo-init-eval.tpl

@@ -73,6 +73,12 @@ db.$MONGODB_COLLECTION_NAME.createIndex({
   'healthevent.entitiesimpacted.entityvalue': 1,
   'healthevent.generatedtimestamp.seconds': 1
 });
+db.$MONGODB_COLLECTION_NAME.createIndex({
+  'healthevent.checkname': 1,
+  'healthevent.nodename': 1,
+  'createdAt': -1,
+  'healthevent.agent': 1
+});
 
 {{- if eq $authMechanism "x509" }}
 // X.509 user creation (only for x509 auth mechanism)

distros/kubernetes/nvsentinel/values-tilt-postgresql.yaml

@@ -246,6 +246,7 @@ postgresql:
               OR document->'healtheventstatus'->>'faultquarantinerecovery' = ''
           );
           CREATE INDEX IF NOT EXISTS idx_health_events_updated_desc ON health_events(updated_at DESC);
+          CREATE INDEX IF NOT EXISTS idx_health_events_analyzer_lookup ON health_events (node_name, event_type, created_at DESC, (document->'healthevent'->>'agent'));
 
           -- GIN index for flexible JSON querying
           CREATE INDEX IF NOT EXISTS idx_health_events_document_gin ON health_events USING GIN (document);

docs/METRICS.md

@@ -10,6 +10,7 @@ This document outlines all Prometheus metrics exposed by NVSentinel components.
 - [Labeler Module](#labeler)
 - [Janitor](#janitor)
 - [Platform Connectors](#platform-connectors)
+- [Health Events Analyzer](#health-events-analyzer)
 - [Health Monitors](#health-monitors)
   - [GPU Health Monitor](#gpu-health-monitor)
   - [Syslog Health Monitor](#syslog-health-monitor)
@@ -237,6 +238,24 @@ These metrics track the internal ring buffer workqueue performance:
 
 ---
 
+## Health Events Analyzer
+
+| Metric Name | Type | Labels | Description |
+|------------|------|--------|-------------|
+| `health_event_analyzer_events_received_total` | Counter | `node_name` | Total analyzer input events received from the watcher |
+| `health_event_analyzer_events_successfully_processed_total` | Counter | - | Total analyzer input events processed successfully |
+| `health_event_analyzer_event_processing_errors` | Counter | `error_type` | Total analyzer processing errors |
+| `health_event_analyzer_event_handling_duration_seconds` | Histogram | - | Analyzer event handling duration, with buckets from 100 ms through about 205 s |
+| `mongo_query_execution_duration_seconds` | Histogram | `rule_name` | Rule aggregation duration for either supported datastore provider |
+| `rule_matched_total` | Counter | `rule_name`, `node_name` | Total matching rule evaluations |
+| `fatal_events_published_total` | Counter | `entity_value` | Total derived unhealthy events published |
+| `health_event_analyzer_recovery_events_published_total` | Counter | `rule_name`, `scope` | Total derived healthy transitions published by recovery-enabled rules |
+| `health_event_analyzer_recovery_persistence_timeouts_total` | Counter | `rule_name`, `state` | Total derived transitions not observed in the store before the persistence deadline |
+| `health_event_analyzer_recovery_stored_document_decode_errors_total` | Counter | `rule_name`, `lookup`, `classification` | Stored health event decode or identity failures observed during recovery lookups, including out-of-scope rows; repeated observations within one persistence wait are counted once |
+| `fatal_event_publishing_errors` | Counter | `error_type` | Total gRPC publication errors |
+
+---
+
 ## Health Monitors
 
 ### GPU Health Monitor

docs/configuration/health-events-analyzer.md

@@ -154,11 +154,97 @@ stage = [
 ]
 ```
 
+Configuration is decoded strictly. Unknown TOML keys, malformed JSON stages,
+and stages containing zero or multiple aggregation operators fail startup instead
+of silently changing rule behavior.
+
+Upgrade note: custom keys that older releases ignored now prevent analyzer startup.
+Remove or correct unknown keys before deploying this version.
+
 The full default ruleset — including all aggregation pipeline stage definitions — is in the chart's `values.yaml` at `distros/kubernetes/nvsentinel/charts/health-events-analyzer/values.yaml`. Refer to that file when writing or reviewing custom rules.
 
+### Derived-condition recovery
+
+The design rationale, tradeoffs, and alternatives are documented in
+[ADR-053](../designs/053-derived-condition-recovery.md).
+
+Rules may opt into automatic recovery by mapping a verified healthy source event
+to the derived condition:
+
+```toml
+[[rules]]
+name = "RepeatedXID94OnSameGPU"
+description = "Repeated XID 94 events on one GPU"
+recommended_action = "CONTACT_SUPPORT"
+message = "Repeated XID 94"
+evaluate_rule = true
+stage = [
+  '{ "$match": { "healthevent.checkname": "SysLogsXIDError", "healthevent.ishealthy": false } }',
+  '{ "$count": "count" }',
+  '{ "$match": { "count": { "$gte": 3 } } }'
+]
+
+[rules.recovery]
+source_agent = "syslog-health-monitor"
+source_check_name = "SysLogsXIDError"
+scope = "entity"
+entity_types = ["GPU_UUID"]
+```
+
+`source_check_name` and `scope` are required. `source_agent` is optional; omit it
+only when more than one trusted producer may publish the recovery event.
+The analyzer rejects `source_agent = "health-events-analyzer"` because analyzer
+output is excluded from its input stream.
+`source_error_codes` is also optional. Set it only when the healthy source event
+carries a code that identifies the recovery; successful GPU-reset events do not.
+When configured, at least one listed code must be present. Entity scope requires one or more
+`entity_types`; node scope must not set `entity_types`. Each configured entity type must have
+exactly one value in an entity-scoped event.
+
+The analyzer publishes a derived healthy event only when the latest derived state
+for the same rule, node, and configured entity set is unhealthy. The event uses
+the rule name as `checkName`, sets `isHealthy=true`, `isFatal=false`, and
+`recommendedAction=NONE`, and leaves the final uncordon decision to
+fault-quarantine. Replayed recovery events therefore converge without repeatedly
+clearing an already-healthy condition. For entity-scoped rules, derived unhealthy
+and healthy events contain only the configured entity types, so both transitions
+address the same downstream fault keys. A matching healthy source with no entities
+is node-wide and clears each active entity-scoped condition for that rule and node;
+a source with only some configured entity types is rejected. If a matching rule
+input lacks a required entity type, the analyzer still publishes the derived fault
+but leaves that event on the existing manual-recovery path.
+
+For recovery-enabled rules, the analyzer normally does not advance a source event's
+resume token until its matching derived transition is visible in the event store;
+the deterministic stored-record exception is described below. If the
+platform connector accepts but drops the queued event before storage, the
+analyzer republishes it. This applies to both unhealthy and healthy transitions,
+so a recovery cannot overtake an earlier derived fault. A delayed healthy event
+never clears a derived fault with a newer generation time. If the transition is
+still not visible after two minutes, the processor exits without acknowledging the
+source. The watcher replays the source after restart instead of blocking the event
+stream indefinitely.
+
+Deterministic failures tied to a rule or stored record are logged, checkpointed,
+and skipped so a poison event cannot halt every later event. Transient datastore
+and publisher failures are not checkpointed and still stop processing for replay.
+
+The persisted source recovery event also becomes the rule's history boundary.
+Later evaluations exclude records stored or generated at or before that event,
+so pre-recovery history and delayed old records cannot immediately recreate the
+condition. Existing derived events do not require migration: state matching uses
+their rule, node, and entity fields.
+
+Recovery is disabled when `evaluate_rule=false`. Healthy events using
+`STORE_ONLY` are not analyzer inputs. Rules without a `[rules.recovery]` block
+retain manual-recovery behavior. The watcher is process-wide, not per-rule: once
+any enabled rule has a recovery mapping, every rule shares the widened watcher
+that also admits healthy events. Healthy events are still offered only to
+recovery mappings, so non-recovery rules never evaluate them.
+
 ### MultipleRemediations Rule
 
-The `MultipleRemediations` rule fires when five or more remediations have been performed on the same node within the preceding 7 days. Unlike other rules, **it applies a node condition that NVSentinel does not automatically clear**, because the rule does not emit healthy events.
+The `MultipleRemediations` rule fires when five or more remediations have been performed on the same node within the preceding 7 days. Its default configuration has no recovery mapping, so **it applies a node condition that NVSentinel does not automatically clear**.
 
 After the underlying hardware issue is resolved, remove the condition manually:
 

docs/designs/031-OTEL-traces.md

@@ -574,7 +574,7 @@ client: &http.Client{
 | Aggregation pipeline (MongoDB) | `analyzer.mongo.aggregate` child span; `analyzer.mongo.rule_name`, `analyzer.mongo.pipeline.documents_matched` (int); DB latency tracked automatically by store client span (`db.aggregate`, `db.duration_ms`) | How long did the pipeline take? How many documents matched? |
 | Event publication (matched rule) | `analyzer.publish_matched_event` child span; `analyzer.event.rule_name`, `analyzer.event.recommended_action`, `analyzer.event.published` (bool) | Was a matched event published? Which rule triggered it? |
 | gRPC call with retry | `analyzer.grpc.publish` child span; `analyzer.grpc.retry_count` (int), `analyzer.grpc.duration_ms` (float), `analyzer.grpc.status` = "success"/"failure" | How many retries were needed for the gRPC call? What was the latency? |
-| XID detector handling | `analyzer.xid.handle` child span; `analyzer.xid.node`, `analyzer.xid.component_class`, `analyzer.xid.is_healthy`, `analyzer.xid.history_cleared` (bool), `analyzer.xid.burst_detected` (bool) | Was XID history cleared? Was a burst detected? |
+| XID detector handling | `health_events_analyzer.handle_xid_detector` child span; `health_events_analyzer.published_event` (bool) | Did XID handling publish a derived event? |
 | XID burst detection | `analyzer.xid.burst_detection` child span; `analyzer.xid.node`, `analyzer.xid.error_code`, `analyzer.xid.burst_detected` (bool), `analyzer.xid.burst_count` (int), `analyzer.event.published` (bool), `analyzer.event.published_rule` | Was an XID burst detected? How many bursts? Which XID code? |
 | Errors | `analyzer.error.type`, `analyzer.error.message` on relevant spans | What went wrong in the analyzer and why? |
 
@@ -794,4 +794,4 @@ OpenTelemetry[https://opentelemetry.io/docs/]
 OpenTelemetry Collector[https://github.com/open-telemetry/opentelemetry-collector]
 Alloy Collector[https://grafana.com/oss/alloy-opentelemetry-collector/]
 Tracing Guide[https://vfunction.com/blog/opentelemetry-tracing-guide/]
-OTEL Logging[https://opentelemetry.io/docs/specs/otel/logs/]
+OTEL Logging[https://opentelemetry.io/docs/specs/otel/logs/]