diff --git a/commons/pkg/configmanager/loader.go b/commons/pkg/configmanager/loader.go index 76670b63a..da77cc6d9 100644 --- a/commons/pkg/configmanager/loader.go +++ b/commons/pkg/configmanager/loader.go @@ -16,6 +16,7 @@ package configmanager import ( "fmt" + "strings" "github.com/BurntSushi/toml" ) @@ -52,3 +53,24 @@ func LoadTOMLConfig[T any](path string, config *T) error { return nil } + +// LoadTOMLConfigStrict loads TOML and rejects keys that are not represented in +// the destination struct. Use it where a misspelled key must fail startup. +func LoadTOMLConfigStrict[T any](path string, config *T) error { + metadata, err := toml.DecodeFile(path, config) + if err != nil { + return fmt.Errorf("failed to decode TOML file %s: %w", path, err) + } + + undecoded := metadata.Undecoded() + if len(undecoded) == 0 { + return nil + } + + keys := make([]string, 0, len(undecoded)) + for _, key := range undecoded { + keys = append(keys, key.String()) + } + + return fmt.Errorf("unknown TOML keys in %s: %s", path, strings.Join(keys, ", ")) +} diff --git a/commons/pkg/configmanager/loader_test.go b/commons/pkg/configmanager/loader_test.go index ceeb40967..dd36a398c 100644 --- a/commons/pkg/configmanager/loader_test.go +++ b/commons/pkg/configmanager/loader_test.go @@ -17,6 +17,7 @@ package configmanager import ( "os" "path/filepath" + "strings" "testing" ) @@ -93,3 +94,50 @@ enabled = true t.Fatal("expected error for invalid TOML syntax, got nil") } } + +func TestLoadTOMLConfigStrictRejectsUnknownKeys(t *testing.T) { + t.Parallel() + + configPath := filepath.Join(t.TempDir(), "config.toml") + contents := "name = \"test\"\nunknown_option = true\n" + if err := os.WriteFile(configPath, []byte(contents), 0o600); err != nil { + t.Fatal(err) + } + + var cfg testTOMLConfig + err := LoadTOMLConfigStrict(configPath, &cfg) + if err == nil || !strings.Contains(err.Error(), "unknown_option") { + t.Fatalf("LoadTOMLConfigStrict() error = %v", err) + } +} + +func TestLoadTOMLConfigStrictAcceptsKnownKeys(t *testing.T) { + t.Parallel() + + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(configPath, []byte("name = \"test\"\nport = 8080\n"), 0o600); err != nil { + t.Fatal(err) + } + + var cfg testTOMLConfig + if err := LoadTOMLConfigStrict(configPath, &cfg); err != nil { + t.Fatalf("LoadTOMLConfigStrict() error = %v", err) + } + if cfg.Name != "test" || cfg.Port != 8080 { + t.Fatalf("LoadTOMLConfigStrict() config = %#v", cfg) + } +} + +func TestLoadTOMLConfigStrictRejectsInvalidTOML(t *testing.T) { + t.Parallel() + + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := os.WriteFile(configPath, []byte("name = [invalid"), 0o600); err != nil { + t.Fatal(err) + } + + var cfg testTOMLConfig + if err := LoadTOMLConfigStrict(configPath, &cfg); err == nil { + t.Fatal("LoadTOMLConfigStrict() accepted invalid TOML") + } +} diff --git a/distros/kubernetes/nvsentinel/charts/mongodb-store/templates/_init-eval.tpl b/distros/kubernetes/nvsentinel/charts/mongodb-store/templates/_init-eval.tpl index be6b94add..d339e0838 100644 --- a/distros/kubernetes/nvsentinel/charts/mongodb-store/templates/_init-eval.tpl +++ b/distros/kubernetes/nvsentinel/charts/mongodb-store/templates/_init-eval.tpl @@ -81,6 +81,12 @@ db.$MONGODB_COLLECTION_NAME.createIndex({ 'healthevent.entitiesimpacted.entityvalue': 1, 'healthevent.generatedtimestamp.seconds': 1 }); +db.$MONGODB_COLLECTION_NAME.createIndex({ + 'healthevent.checkname': 1, + 'healthevent.nodename': 1, + 'createdAt': -1, + 'healthevent.agent': 1 +}); {{- if .Values.mongodb.tls.enabled }} // Create X.509 users (TLS only) var userExists = db.getSiblingDB('\$external').getUser('$MONGODB_APPLICATION_USER_DN'); diff --git a/distros/kubernetes/nvsentinel/templates/_external-mongo-init-eval.tpl b/distros/kubernetes/nvsentinel/templates/_external-mongo-init-eval.tpl index e636d2c27..cec4ed9cc 100644 --- a/distros/kubernetes/nvsentinel/templates/_external-mongo-init-eval.tpl +++ b/distros/kubernetes/nvsentinel/templates/_external-mongo-init-eval.tpl @@ -73,6 +73,12 @@ db.$MONGODB_COLLECTION_NAME.createIndex({ 'healthevent.entitiesimpacted.entityvalue': 1, 'healthevent.generatedtimestamp.seconds': 1 }); +db.$MONGODB_COLLECTION_NAME.createIndex({ + 'healthevent.checkname': 1, + 'healthevent.nodename': 1, + 'createdAt': -1, + 'healthevent.agent': 1 +}); {{- if eq $authMechanism "x509" }} // X.509 user creation (only for x509 auth mechanism) diff --git a/distros/kubernetes/nvsentinel/values-tilt-postgresql.yaml b/distros/kubernetes/nvsentinel/values-tilt-postgresql.yaml index 3d7ea67ed..01ed0f0dc 100644 --- a/distros/kubernetes/nvsentinel/values-tilt-postgresql.yaml +++ b/distros/kubernetes/nvsentinel/values-tilt-postgresql.yaml @@ -246,6 +246,7 @@ postgresql: OR document->'healtheventstatus'->>'faultquarantinerecovery' = '' ); CREATE INDEX IF NOT EXISTS idx_health_events_updated_desc ON health_events(updated_at DESC); + CREATE INDEX IF NOT EXISTS idx_health_events_analyzer_lookup ON health_events (node_name, event_type, created_at DESC, (document->'healthevent'->>'agent')); -- GIN index for flexible JSON querying CREATE INDEX IF NOT EXISTS idx_health_events_document_gin ON health_events USING GIN (document); diff --git a/docs/METRICS.md b/docs/METRICS.md index 308cb16f1..bdd1667e6 100644 --- a/docs/METRICS.md +++ b/docs/METRICS.md @@ -10,6 +10,7 @@ This document outlines all Prometheus metrics exposed by NVSentinel components. - [Labeler Module](#labeler) - [Janitor](#janitor) - [Platform Connectors](#platform-connectors) +- [Health Events Analyzer](#health-events-analyzer) - [Health Monitors](#health-monitors) - [GPU Health Monitor](#gpu-health-monitor) - [Syslog Health Monitor](#syslog-health-monitor) @@ -237,6 +238,24 @@ These metrics track the internal ring buffer workqueue performance: --- +## Health Events Analyzer + +| Metric Name | Type | Labels | Description | +|------------|------|--------|-------------| +| `health_event_analyzer_events_received_total` | Counter | `node_name` | Total analyzer input events received from the watcher | +| `health_event_analyzer_events_successfully_processed_total` | Counter | - | Total analyzer input events processed successfully | +| `health_event_analyzer_event_processing_errors` | Counter | `error_type` | Total analyzer processing errors | +| `health_event_analyzer_event_handling_duration_seconds` | Histogram | - | Analyzer event handling duration, with buckets from 100 ms through about 205 s | +| `mongo_query_execution_duration_seconds` | Histogram | `rule_name` | Rule aggregation duration for either supported datastore provider | +| `rule_matched_total` | Counter | `rule_name`, `node_name` | Total matching rule evaluations | +| `fatal_events_published_total` | Counter | `entity_value` | Total derived unhealthy events published | +| `health_event_analyzer_recovery_events_published_total` | Counter | `rule_name`, `scope` | Total derived healthy transitions published by recovery-enabled rules | +| `health_event_analyzer_recovery_persistence_timeouts_total` | Counter | `rule_name`, `state` | Total derived transitions not observed in the store before the persistence deadline | +| `health_event_analyzer_recovery_stored_document_decode_errors_total` | Counter | `rule_name`, `lookup`, `classification` | Stored health event decode or identity failures observed during recovery lookups, including out-of-scope rows; repeated observations within one persistence wait are counted once | +| `fatal_event_publishing_errors` | Counter | `error_type` | Total gRPC publication errors | + +--- + ## Health Monitors ### GPU Health Monitor diff --git a/docs/configuration/health-events-analyzer.md b/docs/configuration/health-events-analyzer.md index bd8ed8aff..131c3003a 100644 --- a/docs/configuration/health-events-analyzer.md +++ b/docs/configuration/health-events-analyzer.md @@ -154,11 +154,97 @@ stage = [ ] ``` +Configuration is decoded strictly. Unknown TOML keys, malformed JSON stages, +and stages containing zero or multiple aggregation operators fail startup instead +of silently changing rule behavior. + +Upgrade note: custom keys that older releases ignored now prevent analyzer startup. +Remove or correct unknown keys before deploying this version. + The full default ruleset — including all aggregation pipeline stage definitions — is in the chart's `values.yaml` at `distros/kubernetes/nvsentinel/charts/health-events-analyzer/values.yaml`. Refer to that file when writing or reviewing custom rules. +### Derived-condition recovery + +The design rationale, tradeoffs, and alternatives are documented in +[ADR-053](../designs/053-derived-condition-recovery.md). + +Rules may opt into automatic recovery by mapping a verified healthy source event +to the derived condition: + +```toml +[[rules]] +name = "RepeatedXID94OnSameGPU" +description = "Repeated XID 94 events on one GPU" +recommended_action = "CONTACT_SUPPORT" +message = "Repeated XID 94" +evaluate_rule = true +stage = [ + '{ "$match": { "healthevent.checkname": "SysLogsXIDError", "healthevent.ishealthy": false } }', + '{ "$count": "count" }', + '{ "$match": { "count": { "$gte": 3 } } }' +] + +[rules.recovery] +source_agent = "syslog-health-monitor" +source_check_name = "SysLogsXIDError" +scope = "entity" +entity_types = ["GPU_UUID"] +``` + +`source_check_name` and `scope` are required. `source_agent` is optional; omit it +only when more than one trusted producer may publish the recovery event. +The analyzer rejects `source_agent = "health-events-analyzer"` because analyzer +output is excluded from its input stream. +`source_error_codes` is also optional. Set it only when the healthy source event +carries a code that identifies the recovery; successful GPU-reset events do not. +When configured, at least one listed code must be present. Entity scope requires one or more +`entity_types`; node scope must not set `entity_types`. Each configured entity type must have +exactly one value in an entity-scoped event. + +The analyzer publishes a derived healthy event only when the latest derived state +for the same rule, node, and configured entity set is unhealthy. The event uses +the rule name as `checkName`, sets `isHealthy=true`, `isFatal=false`, and +`recommendedAction=NONE`, and leaves the final uncordon decision to +fault-quarantine. Replayed recovery events therefore converge without repeatedly +clearing an already-healthy condition. For entity-scoped rules, derived unhealthy +and healthy events contain only the configured entity types, so both transitions +address the same downstream fault keys. A matching healthy source with no entities +is node-wide and clears each active entity-scoped condition for that rule and node; +a source with only some configured entity types is rejected. If a matching rule +input lacks a required entity type, the analyzer still publishes the derived fault +but leaves that event on the existing manual-recovery path. + +For recovery-enabled rules, the analyzer normally does not advance a source event's +resume token until its matching derived transition is visible in the event store; +the deterministic stored-record exception is described below. If the +platform connector accepts but drops the queued event before storage, the +analyzer republishes it. This applies to both unhealthy and healthy transitions, +so a recovery cannot overtake an earlier derived fault. A delayed healthy event +never clears a derived fault with a newer generation time. If the transition is +still not visible after two minutes, the processor exits without acknowledging the +source. The watcher replays the source after restart instead of blocking the event +stream indefinitely. + +Deterministic failures tied to a rule or stored record are logged, checkpointed, +and skipped so a poison event cannot halt every later event. Transient datastore +and publisher failures are not checkpointed and still stop processing for replay. + +The persisted source recovery event also becomes the rule's history boundary. +Later evaluations exclude records stored or generated at or before that event, +so pre-recovery history and delayed old records cannot immediately recreate the +condition. Existing derived events do not require migration: state matching uses +their rule, node, and entity fields. + +Recovery is disabled when `evaluate_rule=false`. Healthy events using +`STORE_ONLY` are not analyzer inputs. Rules without a `[rules.recovery]` block +retain manual-recovery behavior. The watcher is process-wide, not per-rule: once +any enabled rule has a recovery mapping, every rule shares the widened watcher +that also admits healthy events. Healthy events are still offered only to +recovery mappings, so non-recovery rules never evaluate them. + ### MultipleRemediations Rule -The `MultipleRemediations` rule fires when five or more remediations have been performed on the same node within the preceding 7 days. Unlike other rules, **it applies a node condition that NVSentinel does not automatically clear**, because the rule does not emit healthy events. +The `MultipleRemediations` rule fires when five or more remediations have been performed on the same node within the preceding 7 days. Its default configuration has no recovery mapping, so **it applies a node condition that NVSentinel does not automatically clear**. After the underlying hardware issue is resolved, remove the condition manually: diff --git a/docs/designs/031-OTEL-traces.md b/docs/designs/031-OTEL-traces.md index 861c92952..a0088a61b 100644 --- a/docs/designs/031-OTEL-traces.md +++ b/docs/designs/031-OTEL-traces.md @@ -574,7 +574,7 @@ client: &http.Client{ | Aggregation pipeline (MongoDB) | `analyzer.mongo.aggregate` child span; `analyzer.mongo.rule_name`, `analyzer.mongo.pipeline.documents_matched` (int); DB latency tracked automatically by store client span (`db.aggregate`, `db.duration_ms`) | How long did the pipeline take? How many documents matched? | | Event publication (matched rule) | `analyzer.publish_matched_event` child span; `analyzer.event.rule_name`, `analyzer.event.recommended_action`, `analyzer.event.published` (bool) | Was a matched event published? Which rule triggered it? | | gRPC call with retry | `analyzer.grpc.publish` child span; `analyzer.grpc.retry_count` (int), `analyzer.grpc.duration_ms` (float), `analyzer.grpc.status` = "success"/"failure" | How many retries were needed for the gRPC call? What was the latency? | -| XID detector handling | `analyzer.xid.handle` child span; `analyzer.xid.node`, `analyzer.xid.component_class`, `analyzer.xid.is_healthy`, `analyzer.xid.history_cleared` (bool), `analyzer.xid.burst_detected` (bool) | Was XID history cleared? Was a burst detected? | +| XID detector handling | `health_events_analyzer.handle_xid_detector` child span; `health_events_analyzer.published_event` (bool) | Did XID handling publish a derived event? | | XID burst detection | `analyzer.xid.burst_detection` child span; `analyzer.xid.node`, `analyzer.xid.error_code`, `analyzer.xid.burst_detected` (bool), `analyzer.xid.burst_count` (int), `analyzer.event.published` (bool), `analyzer.event.published_rule` | Was an XID burst detected? How many bursts? Which XID code? | | Errors | `analyzer.error.type`, `analyzer.error.message` on relevant spans | What went wrong in the analyzer and why? | @@ -794,4 +794,4 @@ OpenTelemetry[https://opentelemetry.io/docs/] OpenTelemetry Collector[https://github.com/open-telemetry/opentelemetry-collector] Alloy Collector[https://grafana.com/oss/alloy-opentelemetry-collector/] Tracing Guide[https://vfunction.com/blog/opentelemetry-tracing-guide/] -OTEL Logging[https://opentelemetry.io/docs/specs/otel/logs/] \ No newline at end of file +OTEL Logging[https://opentelemetry.io/docs/specs/otel/logs/] diff --git a/docs/designs/053-derived-condition-recovery.md b/docs/designs/053-derived-condition-recovery.md new file mode 100644 index 000000000..5c40aa30e --- /dev/null +++ b/docs/designs/053-derived-condition-recovery.md @@ -0,0 +1,219 @@ +# ADR-053: Health Events Analyzer — Derived-Condition Recovery + +## Context + +The health-events-analyzer turns a history of source health events into synthetic +derived conditions. A derived unhealthy event uses the rule name as its +`checkName` and can flow through the same quarantine, drain, and remediation +pipeline as an event emitted directly by a health monitor. + +Before this decision, those derived conditions had no automatic inverse +transition. A source health monitor could later report that the underlying +condition was healthy, but that event named the source check rather than the +derived rule. The analyzer therefore had no explicit basis for deciding which +derived condition, node, or device identity the healthy event should clear. +Operators had to clear the condition manually or keep analyzer output in +`STORE_ONLY` mode to avoid a permanently latched node condition. + +Automatic recovery must account for several constraints: + +- A rule may represent either one node-wide condition or separate conditions + for entities such as GPUs. +- Change-stream events can be replayed after restart, and provider ordering is + not a sufficient idempotency boundary by itself. +- Delayed records from before a recovery must not immediately recreate the + recovered condition. +- MongoDB and PostgreSQL must implement the same filter and state semantics. +- Analyzer-produced events must never be consumed as new analyzer inputs. +- A transient store or publish failure must remain replayable, while one + malformed record must not permanently block the shared stream. + +## Decision + +Add opt-in, rule-specific recovery mappings to the health-events-analyzer. A +mapping explicitly identifies a trusted healthy source event and selects either +node or entity scope. The analyzer publishes a derived healthy transition only +when persisted state shows that the same rule and recovery identity are +currently unhealthy. + +Recovery is not inferred from an aggregation pipeline. Rules without a recovery +mapping retain manual-recovery behavior. + +## Implementation + +### Recovery configuration + +An enabled rule may define a `[rules.recovery]` block with: + +- `source_check_name`, which is required; +- optional `source_agent` and `source_error_codes` constraints; +- `scope = "node"` or `scope = "entity"`; and +- `entity_types` for entity scope. + +Configuration validation rejects empty or duplicate values, entity types on a +node-scoped mapping, an entity-scoped mapping without entity types, and +`source_agent = "health-events-analyzer"`. The analyzer's own output is excluded +from input, so accepting it as a recovery source would create an unreachable and +misleading configuration. + +### Recovery identity and transition + +The identity of a derived condition is the rule name, node name, and, for entity +scope, the configured set of entity type/value pairs. Entity-scoped source +events must provide exactly one value for every configured entity type. A +matching source event without entity values acts as a node-wide recovery and +clears each active entity identity for that rule and node. A partially specified +entity identity is rejected rather than broadened. + +For every resolved identity, the analyzer reads the latest persisted derived +state. It publishes a transition only when that state is unhealthy. The derived +healthy event: + +- uses the rule name as `checkName`; +- sets `isHealthy=true`, `isFatal=false`, and `recommendedAction=NONE`; +- preserves the rule's processing strategy; and +- carries the same configured entity identity as the derived fault. + +This state check makes replay converge without repeatedly publishing clears for +an identity that is already healthy. The analyzer leaves uncordon and other +downstream policy decisions to fault-quarantine. + +### Ordering, persistence, and replay + +For recovery-enabled rules, the analyzer normally advances a source event's +resume token only after the corresponding derived transition is visible in the +event store. This applies to both the unhealthy transition and its later healthy +transition, preventing a recovery from overtaking an unpersisted derived fault. + +The persisted recovery source becomes the rule's history boundary. Later rule +evaluation excludes records stored or generated at or before that boundary, so +delayed pre-recovery history cannot immediately recreate the condition. + +Transient datastore and publisher errors leave the source unacknowledged and +stop processing so the change stream can replay it. Confirmation is bounded by +a two-minute deadline; expiration exits processing without acknowledging the +source rather than blocking the stream forever. Deterministic configuration or +stored-record failures are logged, counted, and checkpointed after applying the +narrowest safe recovery holdback, so poison data does not halt unrelated rules +and identities. + +### Input and datastore behavior + +When any enabled rule has a recovery mapping, the shared analyzer watcher also +admits processable healthy source events. Healthy events are considered only by +recovery mappings and are not evaluated as ordinary failure inputs. Both the +watcher filter and every rule pipeline exclude events whose agent is +`health-events-analyzer`, preventing feedback loops. + +MongoDB and PostgreSQL implement equivalent recovery filters and deterministic +query-error handling. Provider-specific lookup indexes support state and history +queries. PostgreSQL adds its upgrade index through a non-fatal background task: +startup does not wait for a concurrent index build, shutdown cancels and joins +the task, and a missing or invalid index is retried on a later startup. + +## Rationale + +- **Explicit semantics:** A rule author, rather than a heuristic, defines which + healthy signal is authoritative for a derived condition. +- **Scoped safety:** Node and entity identities prevent one device recovery from + clearing unrelated active faults. +- **Replay convergence:** Persisted state and history boundaries make duplicate + and delayed events idempotent across restarts. +- **Provider parity:** The decision is expressed in datastore-independent rule + and identity semantics, with provider-specific query implementations. +- **Pipeline separation:** The analyzer publishes normal health events and does + not directly mutate Kubernetes node status or remediation state. + +## Consequences + +### Positive + +- Derived conditions can clear automatically when an explicitly trusted source + reports recovery. +- Existing rules remain unchanged unless they opt into recovery. +- Entity-scoped faults recover independently while node-wide recovery remains + available when a source cannot identify individual entities. +- Replay, restart, and delayed-history behavior is defined and testable. +- Recovery events continue through the standard storage, quarantine, and + remediation pipeline instead of introducing a second mutation path. + +### Negative + +- Rule configuration becomes more complex and an incorrect source mapping can + prevent a legitimate recovery from matching. +- Enabling recovery for one rule widens the process-wide watcher input for all + rules, increasing read and dispatch work. +- State confirmation and boundary queries add datastore load and make recovery + eventually consistent rather than instantaneous. +- Deterministically malformed stored records may conservatively withhold a + boundary or recovery identity until operators repair or remove the record. +- Persistence of the derived healthy event does not itself guarantee that every + downstream side effect succeeded. In particular, a terminal Kubernetes API + write failure can still leave a node condition latched until the connector + retry work in issue #1743 is deployed. + +### Mitigations + +- Validate mappings strictly at startup and keep recovery opt-in per rule. +- Use identity-scoped holdbacks so one malformed entity does not block unrelated + recoveries when its identity can be determined safely. +- Bound persistence confirmation, preserve unacknowledged sources for replay, + and expose recovery outcome and failure metrics. +- Maintain provider-specific lookup indexes without making performance-index + creation a startup requirement. +- Address downstream Kubernetes write reliability independently in #1743 so + the recovery feature and connector bug can be reviewed and released + separately. + +## Alternatives Considered + +### Keep all derived conditions on manual recovery + +**Rejected** because: it leaves operators responsible for clearing analyzer +conditions and makes `EXECUTE_REMEDIATION` unsafe for conditions that can later +become healthy. It also preserves the fleet workaround of using `STORE_ONLY` +for otherwise actionable derived events. + +### Infer recovery automatically from the rule pipeline + +**Rejected** because: aggregation pipelines describe how to detect a historical +failure, not which future source is authoritative for clearing it. Inverting an +arbitrary pipeline is ambiguous, especially for count, time-window, and +multi-source rules. + +### Have the analyzer patch Kubernetes conditions directly + +**Rejected** because: it would bypass storage, fault-quarantine policy, tracing, +and the existing platform-connector path. It would also couple analyzer rules to +Kubernetes and produce different behavior for non-Kubernetes deployments. + +### Mutate or delete the stored derived fault + +**Rejected** because: health events are an append-only history. Rewriting the +fault would erase audit context and would not produce the healthy transition +consumed by downstream components. + +### Periodically scan and reconcile every derived condition + +**Rejected** because: polling adds continuous datastore load and still requires +an explicit definition of the healthy source. Change-stream replay plus a +persisted boundary provides recovery without a second scheduler. + +## Notes + +- This ADR does not enable recovery for every existing analyzer rule. Each rule + owner must select and validate an authoritative source mapping. +- This ADR does not change fault-quarantine uncordon policy or directly repair + downstream connector delivery failures. +- The ordered Kubernetes retry fix for #1743 is intentionally maintained in a + separate pull request from the recovery feature. + +## References + +- [Issue #1553: Health-events-analyzer recovery](https://github.com/NVIDIA/NVSentinel/issues/1553) +- [Issue #1743: Kubernetes connector drops failed writes](https://github.com/NVIDIA/NVSentinel/issues/1743) +- [ADR-006: Platform Connector Event Buffering](./006-platform-connector-reliability.md) +- [ADR-007: Health Event Correlation](./007-event-correlation-and-analysis.md) +- [ADR-025: Processing Strategy for Health Checks](./025-processing-strategy-for-health-checks.md) +- [ADR-039: Health Event Deduplication](./039-health-event-deduplication.md) +- [Health-events-analyzer configuration](../configuration/health-events-analyzer.md#derived-condition-recovery) diff --git a/docs/health-events-analyzer.md b/docs/health-events-analyzer.md index c64a30def..359f8319f 100644 --- a/docs/health-events-analyzer.md +++ b/docs/health-events-analyzer.md @@ -23,11 +23,12 @@ The Health Events Analyzer consumes MongoDB change stream events from the health 2. **Runs aggregation pipelines** over configurable time windows (hours to days) to look for patterns across events on the same node or GPU 3. **Evaluates rules** shipped as TOML-encoded MongoDB aggregation stages in the Helm `config:` block; each rule targets a specific pattern (e.g., repeated failures, die-level clustering, XID 74 register decoding, multiple remediations) 4. **Emits synthetic events** when a rule matches: each derived event carries its own `checkName` (the rule name) and flows through the standard Fault Quarantine → Node Drainer → Fault Remediation pipeline exactly as if a health monitor had reported it -5. **Applies the configured processing strategy** — `EXECUTE_REMEDIATION` for live operation or `STORE_ONLY` for shadow-mode observation with no side effects +5. **Recovers configured derived conditions** when a rule-specific healthy source event arrives, using node or entity scope and a persisted history boundary +6. **Applies the configured processing strategy** — `EXECUTE_REMEDIATION` for live operation or `STORE_ONLY` for shadow-mode observation with no side effects -The Analyzer does not modify or delete raw events. It only appends new synthetic events into the same pipeline. +The Analyzer does not modify or delete raw events. It only appends new synthetic events into the same pipeline. Recovery is opt-in per rule; rules without a recovery mapping retain manual recovery. Every rule receives the same mandatory admission predicates when no persisted recovery boundary applies. A recovery rule with a boundary adds whichever time and generation guards are available to its first stage. When any enabled rule has a recovery mapping, the shared watcher admits all processable healthy events so the reconciler can select configured recovery sources; this also widens the shared input stream seen by non-recovery rules. This recovery-enabled watcher drops the unhealthy-only predicate on both providers and also admits events whose processing strategy is `UNSPECIFIED`. For a recovery-enabled rule, a source event normally remains unacknowledged until the transition it derives is visible in the store—the derived fault for a matched unhealthy source, the derived healthy transition for a matched recovery source. If a stored document is deterministically malformed or cannot be assigned to a recovery identity, the Analyzer logs and counts the incomplete scan. Any incomplete scan withholds the rule's node-wide boundary. Beyond that, a malformed document whose identity fields decode into a rule-matching identity withholds only that identity's recovery; a document that decodes but yields no rule-matching identity withholds no additional target; and a document whose identity fields cannot be decoded at all withholds every target on the node. The source is checkpointed so poison data cannot halt the stream only when every failure recorded by that scan is deterministic; a scan that also hits a transient decode failure leaves the source unacknowledged for replay. A two-minute deadline prevents a transient connector outage from blocking the stream forever; when the deadline expires, the processor exits and replays the unacknowledged source after restart. Other deterministic rule failures, such as invalid pipeline syntax, are logged and skipped after checkpointing; transient store or publisher failures remain unacknowledged for replay. -**Loop prevention**: The Analyzer excludes its own events at two layers. First, the change-stream ingestion filter drops any event where `agent == "health-events-analyzer"`, so derived events are never re-ingested. Second, every rule's aggregation pipeline opens with a guard `$match` stage that also filters out events produced by `health-events-analyzer`, so the Analyzer never counts its own synthetic events when evaluating rules. +**Loop prevention**: The Analyzer excludes its own events at two layers. First, the change-stream ingestion filter drops any event where `agent == "health-events-analyzer"`, so derived events are never re-ingested. Second, every rule's aggregation pipeline opens with a guard `$match` stage that also filters out events produced by `health-events-analyzer`, so the Analyzer never counts its own synthetic events when evaluating rules. A recovery mapping therefore cannot use `source_agent = "health-events-analyzer"`; configuration validation rejects it at startup. Healthy events are handled only as possible recovery sources and are not evaluated as ordinary failure inputs. The in-memory XID burst detector—which the analyzer enables only on the PostgreSQL provider, since MongoDB deployments detect the same bursts inside the rule pipeline—also receives only unhealthy events; it removes old observations by age rather than through a recovery clearing path. ### XID 74 Register Decoding diff --git a/docs/postgresql-schema.sql b/docs/postgresql-schema.sql index c08f28414..1b1a5f78a 100644 --- a/docs/postgresql-schema.sql +++ b/docs/postgresql-schema.sql @@ -133,6 +133,7 @@ AND ( OR document->'healtheventstatus'->>'faultquarantinerecovery' = '' ); CREATE INDEX IF NOT EXISTS idx_health_events_updated_desc ON health_events(updated_at DESC); +CREATE INDEX IF NOT EXISTS idx_health_events_analyzer_lookup ON health_events (node_name, event_type, created_at DESC, (document->'healthevent'->>'agent')); -- GIN index for flexible JSON querying CREATE INDEX IF NOT EXISTS idx_health_events_document_gin ON health_events USING GIN (document); diff --git a/health-events-analyzer/go.mod b/health-events-analyzer/go.mod index 6aaf995e2..8a3109e96 100644 --- a/health-events-analyzer/go.mod +++ b/health-events-analyzer/go.mod @@ -9,6 +9,7 @@ require ( github.com/nvidia/nvsentinel/store-client v0.0.0 github.com/prometheus/client_golang v1.24.1 github.com/stretchr/testify v1.12.1 + go.mongodb.org/mongo-driver/v2 v2.8.1 go.opentelemetry.io/otel v1.46.0 go.opentelemetry.io/otel/trace v1.46.0 golang.org/x/sync v0.22.0 @@ -50,6 +51,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.19.2 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lib/pq v1.12.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect @@ -65,7 +67,6 @@ require ( github.com/xdg-go/stringprep v1.0.4 // indirect github.com/yandex/protoc-gen-crd v1.1.0 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect - go.mongodb.org/mongo-driver/v2 v2.8.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/go.mongodb.org/mongo-driver/v2/mongo/otelmongo v0.0.0-20260824184942-eee67831109c // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect diff --git a/health-events-analyzer/main.go b/health-events-analyzer/main.go index 96e5aa503..395dac12d 100644 --- a/health-events-analyzer/main.go +++ b/health-events-analyzer/main.go @@ -94,9 +94,13 @@ func loadDatabaseConfig(databaseClientCertMountPath string) (*datastore.DataStor return config, nil } -func createPipeline() any { +func createPipeline(config *config.TomlConfig) any { builder := client.GetPipelineBuilder() - return builder.BuildProcessableNonFatalUnhealthyInsertsPipeline() + if !config.HasEnabledRecovery() { + return builder.BuildProcessableNonFatalUnhealthyInsertsPipeline() + } + + return client.WithExtendedFilters(builder.BuildAnalyzerHealthEventInsertsPipeline()) } func connectToPlatform(socket, tokenPath string, processingStrategy protos.ProcessingStrategy) ( @@ -144,8 +148,6 @@ func run() error { return err } - pipeline := createPipeline() - value, ok := protos.ProcessingStrategy_value[*processingStrategyFlag] if !ok { return fmt.Errorf("unexpected processingStrategy value: %q", *processingStrategyFlag) @@ -165,6 +167,8 @@ func run() error { return fmt.Errorf("error loading TOML config: %w", err) } + pipeline := createPipeline(tomlConfig) + for _, rule := range tomlConfig.Rules { ff.Set(rule.Name, rule.EvaluateRule) } diff --git a/health-events-analyzer/main_test.go b/health-events-analyzer/main_test.go new file mode 100644 index 000000000..393bbf604 --- /dev/null +++ b/health-events-analyzer/main_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/config" + "github.com/nvidia/nvsentinel/store-client/pkg/client" +) + +func TestCreatePipelineRequiresEnabledRecovery(t *testing.T) { + builder := client.GetPipelineBuilder() + + for name, test := range map[string]struct { + config *config.TomlConfig + want any + }{ + "nil config": { + want: builder.BuildProcessableNonFatalUnhealthyInsertsPipeline(), + }, + "manual recovery": { + config: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{{EvaluateRule: true}}}, + want: builder.BuildProcessableNonFatalUnhealthyInsertsPipeline(), + }, + "disabled recovery rule": { + config: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{{ + Recovery: &config.RecoveryMapping{}, + }}}, + want: builder.BuildProcessableNonFatalUnhealthyInsertsPipeline(), + }, + "enabled recovery rule": { + config: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{{ + EvaluateRule: true, + Recovery: &config.RecoveryMapping{}, + }}}, + want: client.WithExtendedFilters(builder.BuildAnalyzerHealthEventInsertsPipeline()), + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, test.want, createPipeline(test.config)) + }) + } +} diff --git a/health-events-analyzer/pkg/analyzer/xid_burst_detector.go b/health-events-analyzer/pkg/analyzer/xid_burst_detector.go index 1e4078b21..f59dc03c3 100644 --- a/health-events-analyzer/pkg/analyzer/xid_burst_detector.go +++ b/health-events-analyzer/pkg/analyzer/xid_burst_detector.go @@ -616,10 +616,8 @@ func (d *XidBurstDetector) GetPerGPUBurstStats() map[string]map[string]int { return stats } -// ClearNodeHistory clears all XID event history for a specific node across -// every GPU. This should be called when a healthy event is received for the -// node, indicating that the GPU issues have been resolved and we should start -// fresh. +// ClearNodeHistory clears all XID event history for a node. It is retained for +// direct detector tests; production history expires by age. func (d *XidBurstDetector) ClearNodeHistory(nodeName string) { d.mu.Lock() defer d.mu.Unlock() diff --git a/health-events-analyzer/pkg/config/rules.go b/health-events-analyzer/pkg/config/rules.go index 6e2515f92..93d26579e 100644 --- a/health-events-analyzer/pkg/config/rules.go +++ b/health-events-analyzer/pkg/config/rules.go @@ -15,11 +15,33 @@ package config import ( + "encoding/json" "fmt" + "strings" "github.com/nvidia/nvsentinel/commons/pkg/configmanager" + protos "github.com/nvidia/nvsentinel/data-models/pkg/protos" ) +type RecoveryScope string + +const ( + RecoveryScopeNode RecoveryScope = "node" + RecoveryScopeEntity RecoveryScope = "entity" + analyzerAgentName = "health-events-analyzer" +) + +// RecoveryMapping identifies the healthy source event that resolves a derived +// condition. Rules without this block retain the existing manual-recovery +// behavior. +type RecoveryMapping struct { + SourceAgent string `toml:"source_agent"` + SourceCheckName string `toml:"source_check_name"` + SourceErrorCodes []string `toml:"source_error_codes"` + Scope RecoveryScope `toml:"scope"` + EntityTypes []string `toml:"entity_types"` +} + type HealthEventsAnalyzerRule struct { Name string `toml:"name"` Description string `toml:"description"` @@ -28,18 +50,138 @@ type HealthEventsAnalyzerRule struct { Message string `toml:"message"` EvaluateRule bool `toml:"evaluate_rule"` // Optional: override the module-level processing strategy for events published by this rule. - ProcessingStrategy string `toml:"processing_strategy"` + ProcessingStrategy string `toml:"processing_strategy"` + Recovery *RecoveryMapping `toml:"recovery"` } type TomlConfig struct { Rules []HealthEventsAnalyzerRule `toml:"rules"` } +func (c *TomlConfig) HasEnabledRecovery() bool { + if c == nil { + return false + } + + for i := range c.Rules { + if c.Rules[i].EvaluateRule && c.Rules[i].Recovery != nil { + return true + } + } + + return false +} + func LoadTomlConfig(path string) (*TomlConfig, error) { var config TomlConfig - if err := configmanager.LoadTOMLConfig(path, &config); err != nil { + if err := configmanager.LoadTOMLConfigStrict(path, &config); err != nil { return nil, fmt.Errorf("failed to decode TOML config from %s: %w", path, err) } + if err := config.Validate(); err != nil { + return nil, fmt.Errorf("invalid health-events-analyzer config: %w", err) + } + return &config, nil } + +func (c *TomlConfig) Validate() error { + for i := range c.Rules { + if err := c.Rules[i].validateProcessingStrategy(); err != nil { + return fmt.Errorf("rule %q: %w", c.Rules[i].Name, err) + } + + if err := c.Rules[i].validateStages(); err != nil { + return fmt.Errorf("rule %q: %w", c.Rules[i].Name, err) + } + + if err := c.Rules[i].validateRecovery(); err != nil { + return fmt.Errorf("rule %q: %w", c.Rules[i].Name, err) + } + } + + return nil +} + +func (r *HealthEventsAnalyzerRule) validateProcessingStrategy() error { + if r.ProcessingStrategy == "" { + return nil + } + + if _, ok := protos.ProcessingStrategy_value[r.ProcessingStrategy]; !ok { + return fmt.Errorf("processing_strategy has invalid value %q", r.ProcessingStrategy) + } + + return nil +} + +func (r *HealthEventsAnalyzerRule) validateStages() error { + for i, stage := range r.Stage { + var parsed map[string]any + if err := json.Unmarshal([]byte(stage), &parsed); err != nil { + return fmt.Errorf("stage %d is not valid JSON: %w", i, err) + } + + if len(parsed) != 1 { + return fmt.Errorf("stage %d must contain exactly one aggregation operator", i) + } + } + + return nil +} + +func (r *HealthEventsAnalyzerRule) validateRecovery() error { + if r.Recovery == nil { + return nil + } + + recovery := r.Recovery + recovery.SourceAgent = strings.TrimSpace(recovery.SourceAgent) + recovery.SourceCheckName = strings.TrimSpace(recovery.SourceCheckName) + + if recovery.SourceCheckName == "" { + return fmt.Errorf("recovery.source_check_name is required") + } + + if recovery.SourceAgent == analyzerAgentName { + return fmt.Errorf("recovery.source_agent %q is excluded from analyzer input", analyzerAgentName) + } + + switch recovery.Scope { + case RecoveryScopeNode: + if len(recovery.EntityTypes) != 0 { + return fmt.Errorf("recovery.entity_types must be empty for node scope") + } + case RecoveryScopeEntity: + if len(recovery.EntityTypes) == 0 { + return fmt.Errorf("recovery.entity_types is required for entity scope") + } + default: + return fmt.Errorf("recovery.scope must be %q or %q", RecoveryScopeNode, RecoveryScopeEntity) + } + + if err := validateUniqueNonEmpty("recovery.entity_types", recovery.EntityTypes); err != nil { + return err + } + + return validateUniqueNonEmpty("recovery.source_error_codes", recovery.SourceErrorCodes) +} + +func validateUniqueNonEmpty(field string, values []string) error { + seen := make(map[string]struct{}, len(values)) + + for i := range values { + values[i] = strings.TrimSpace(values[i]) + if values[i] == "" { + return fmt.Errorf("%s must not contain empty values", field) + } + + if _, exists := seen[values[i]]; exists { + return fmt.Errorf("%s contains duplicate value %q", field, values[i]) + } + + seen[values[i]] = struct{}{} + } + + return nil +} diff --git a/health-events-analyzer/pkg/config/rules_test.go b/health-events-analyzer/pkg/config/rules_test.go new file mode 100644 index 000000000..c292da2ce --- /dev/null +++ b/health-events-analyzer/pkg/config/rules_test.go @@ -0,0 +1,240 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLoadTomlConfigRecovery(t *testing.T) { + path := filepath.Join(t.TempDir(), "rules.toml") + contents := ` +[[rules]] +name = "RepeatedXID94OnSameGPU" +evaluate_rule = true +stage = [] +recommended_action = "CONTACT_SUPPORT" + +[rules.recovery] +source_agent = "syslog-health-monitor" +source_check_name = "SysLogsXIDError" +source_error_codes = ["94"] +scope = "entity" +entity_types = ["GPU_UUID"] +` + require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) + + cfg, err := LoadTomlConfig(path) + require.NoError(t, err) + require.Len(t, cfg.Rules, 1) + require.Equal(t, &RecoveryMapping{ + SourceAgent: "syslog-health-monitor", + SourceCheckName: "SysLogsXIDError", + SourceErrorCodes: []string{"94"}, + Scope: RecoveryScopeEntity, + EntityTypes: []string{"GPU_UUID"}, + }, cfg.Rules[0].Recovery) +} + +func TestLoadTomlConfigRejectsInvalidRecovery(t *testing.T) { + path := filepath.Join(t.TempDir(), "rules.toml") + contents := ` +[[rules]] +name = "invalid-recovery" + +[rules.recovery] +scope = "node" +` + require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) + + config, err := LoadTomlConfig(path) + require.Nil(t, config) + require.ErrorContains(t, err, "invalid health-events-analyzer config") + require.ErrorContains(t, err, "source_check_name is required") +} + +func TestLoadTomlConfigRejectsUnknownKeys(t *testing.T) { + path := filepath.Join(t.TempDir(), "rules.toml") + contents := ` +[[rules]] +name = "misspelled-recovery" +evaluate_rule = true + +[rules.recovery] +source_agent = "syslog-health-monitor" +source_check_name = "SysLogsXIDError" +source_error_code = ["94"] +scope = "node" +` + require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) + + config, err := LoadTomlConfig(path) + require.Nil(t, config) + require.ErrorContains(t, err, "source_error_code") +} + +func TestConfigValidationRejectsInvalidStages(t *testing.T) { + for name, stage := range map[string]string{ + "invalid JSON": `{invalid}`, + "empty stage": `{}`, + "multiple operators": `{"$match": {}, "$count": "count"}`, + } { + t.Run(name, func(t *testing.T) { + config := &TomlConfig{Rules: []HealthEventsAnalyzerRule{{Name: name, Stage: []string{stage}}}} + require.ErrorContains(t, config.Validate(), "stage 0") + }) + } +} + +func TestConfigValidationRejectsInvalidProcessingStrategy(t *testing.T) { + for _, strategy := range []string{"UNSPECIFIED", "EXECUTE_REMEDIATION", "STORE_ONLY", "STORE_AND_ANALYSE"} { + t.Run(strategy, func(t *testing.T) { + config := &TomlConfig{Rules: []HealthEventsAnalyzerRule{{ + Name: "valid-strategy", + ProcessingStrategy: strategy, + }}} + require.NoError(t, config.Validate()) + }) + } + + config := &TomlConfig{Rules: []HealthEventsAnalyzerRule{{ + Name: "invalid-strategy", + ProcessingStrategy: "STORE-ONLY", + }}} + require.ErrorContains(t, config.Validate(), `processing_strategy has invalid value "STORE-ONLY"`) +} + +func TestRecoveryValidationAllowsRulesWithoutMapping(t *testing.T) { + config := &TomlConfig{Rules: []HealthEventsAnalyzerRule{{Name: "manual-recovery"}}} + require.NoError(t, config.Validate()) +} + +func TestHasEnabledRecovery(t *testing.T) { + require.False(t, (*TomlConfig)(nil).HasEnabledRecovery()) + require.False(t, (&TomlConfig{Rules: []HealthEventsAnalyzerRule{{ + EvaluateRule: true, + }}}).HasEnabledRecovery()) + require.False(t, (&TomlConfig{Rules: []HealthEventsAnalyzerRule{{ + Recovery: &RecoveryMapping{}, + }}}).HasEnabledRecovery()) + require.True(t, (&TomlConfig{Rules: []HealthEventsAnalyzerRule{{ + EvaluateRule: true, + Recovery: &RecoveryMapping{}, + }}}).HasEnabledRecovery()) +} + +func TestRecoveryMappingValidation(t *testing.T) { + tests := []struct { + name string + mapping *RecoveryMapping + wantErr string + }{ + { + name: "node scope", + mapping: &RecoveryMapping{ + SourceCheckName: "NodeRecovered", + Scope: RecoveryScopeNode, + }, + }, + { + name: "entity scope", + mapping: &RecoveryMapping{ + SourceCheckName: "GpuRecovered", + Scope: RecoveryScopeEntity, + EntityTypes: []string{"GPU_UUID"}, + }, + }, + { + name: "missing source check", + mapping: &RecoveryMapping{ + Scope: RecoveryScopeNode, + }, + wantErr: "source_check_name is required", + }, + { + name: "analyzer source is unreachable", + mapping: &RecoveryMapping{ + SourceAgent: analyzerAgentName, + SourceCheckName: "Recovered", + Scope: RecoveryScopeNode, + }, + wantErr: "excluded from analyzer input", + }, + { + name: "invalid scope", + mapping: &RecoveryMapping{ + SourceCheckName: "Recovered", + Scope: "cluster", + }, + wantErr: "scope must be", + }, + { + name: "entity scope without entity types", + mapping: &RecoveryMapping{ + SourceCheckName: "Recovered", + Scope: RecoveryScopeEntity, + }, + wantErr: "entity_types is required", + }, + { + name: "node scope with entity types", + mapping: &RecoveryMapping{ + SourceCheckName: "Recovered", + Scope: RecoveryScopeNode, + EntityTypes: []string{"GPU_UUID"}, + }, + wantErr: "entity_types must be empty", + }, + { + name: "duplicate entity type", + mapping: &RecoveryMapping{ + SourceCheckName: "Recovered", + Scope: RecoveryScopeEntity, + EntityTypes: []string{"GPU", "GPU"}, + }, + wantErr: "duplicate value", + }, + { + name: "empty error code", + mapping: &RecoveryMapping{ + SourceCheckName: "Recovered", + Scope: RecoveryScopeNode, + SourceErrorCodes: []string{""}, + }, + wantErr: "must not contain empty values", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := &TomlConfig{Rules: []HealthEventsAnalyzerRule{{ + Name: "derived-condition", + Recovery: test.mapping, + }}} + + err := cfg.Validate() + if test.wantErr == "" { + require.NoError(t, err) + return + } + + require.ErrorContains(t, err, test.wantErr) + }) + } +} diff --git a/health-events-analyzer/pkg/publisher/publisher.go b/health-events-analyzer/pkg/publisher/publisher.go index a5f6c3e61..95d30dc7b 100644 --- a/health-events-analyzer/pkg/publisher/publisher.go +++ b/health-events-analyzer/pkg/publisher/publisher.go @@ -30,15 +30,16 @@ import ( "github.com/nvidia/nvsentinel/commons/pkg/tracing" protos "github.com/nvidia/nvsentinel/data-models/pkg/protos" "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/config" + "github.com/nvidia/nvsentinel/store-client/pkg/client" ) const ( maxRetries int = 5 delay time.Duration = 5 * time.Second - // Carries the triggering event's generated timestamp, which the derived event replaces - // with its own. - sourceGeneratedTimestampMetadataKey = "source_generated_timestamp" + // SourceGeneratedTimestampMetadataKey carries the triggering event's generated timestamp, + // which the derived event replaces with its own. + SourceGeneratedTimestampMetadataKey = "source_generated_timestamp" ) type PublisherConfig struct { @@ -71,7 +72,7 @@ func (p *PublisherConfig) sendHealthEventWithRetry(ctx context.Context, healthEv Jitter: 0.1, } - err := wait.ExponentialBackoff(backoff, func() (bool, error) { + err := wait.ExponentialBackoffWithContext(ctx, backoff, func(ctx context.Context) (bool, error) { _, err := p.platformConnectorClient.HealthEventOccurredV1(ctx, healthEvents) if err == nil { slog.DebugContext(ctx, "Successfully sent health events", "events", healthEvents) @@ -124,21 +125,60 @@ func NewPublisher(platformConnectorClient protos.PlatformConnectorClient, func (p *PublisherConfig) Publish(ctx context.Context, event *protos.HealthEvent, recommendedAction protos.RecommendedAction, ruleName string, message string, rule *config.HealthEventsAnalyzerRule) error { + return p.publish(ctx, event, publishOptions{ + recommendedAction: recommendedAction, + ruleName: ruleName, + message: message, + rule: rule, + }) +} + +// PublishRecovery publishes the healthy transition for a derived condition. +// Error codes are intentionally empty so downstream consumers clear every +// failure for the selected derived condition and entity scope. +func (p *PublisherConfig) PublishRecovery( + ctx context.Context, + event *protos.HealthEvent, + ruleName string, + entities []*protos.Entity, + rule *config.HealthEventsAnalyzerRule, +) error { + return p.publish(ctx, event, publishOptions{ + recommendedAction: protos.RecommendedAction_NONE, + ruleName: ruleName, + message: fmt.Sprintf("Recovered derived condition %s", ruleName), + rule: rule, + isHealthy: true, + entities: entities, + }) +} + +type publishOptions struct { + recommendedAction protos.RecommendedAction + ruleName string + message string + rule *config.HealthEventsAnalyzerRule + isHealthy bool + entities []*protos.Entity +} + +func (p *PublisherConfig) publish(ctx context.Context, event *protos.HealthEvent, options publishOptions) error { ctx, span := tracing.StartSpan(ctx, "health_events_analyzer.publish") defer span.End() span.SetAttributes( - attribute.String("health_events_analyzer.publish.rule_name", ruleName), - attribute.String("health_events_analyzer.publish.recommended_action", recommendedAction.String()), + attribute.String("health_events_analyzer.publish.rule_name", options.ruleName), + attribute.String("health_events_analyzer.publish.recommended_action", options.recommendedAction.String()), + attribute.Bool("health_events_analyzer.publish.is_healthy", options.isHealthy), ) newEvent := proto.Clone(event).(*protos.HealthEvent) newEvent.Agent = "health-events-analyzer" - newEvent.CheckName = ruleName - newEvent.RecommendedAction = recommendedAction - newEvent.IsHealthy = false - newEvent.Message = message + newEvent.CheckName = options.ruleName + newEvent.RecommendedAction = options.recommendedAction + newEvent.IsHealthy = options.isHealthy + newEvent.Message = options.message // The clone inherits the triggering event's timestamp, which dates the derived event to // the original fault rather than to detection. The source value is kept in metadata. @@ -147,7 +187,7 @@ func (p *PublisherConfig) Publish(ctx context.Context, event *protos.HealthEvent newEvent.Metadata = make(map[string]string, 1) } - newEvent.Metadata[sourceGeneratedTimestampMetadataKey] = src.AsTime().UTC().Format(time.RFC3339Nano) + newEvent.Metadata[SourceGeneratedTimestampMetadataKey] = src.AsTime().UTC().Format(time.RFC3339Nano) } newEvent.GeneratedTimestamp = timestamppb.New(time.Now()) @@ -155,25 +195,35 @@ func (p *PublisherConfig) Publish(ctx context.Context, event *protos.HealthEvent // Default from module configuration, with an optional rule-level override. newEvent.ProcessingStrategy = p.processingStrategy - if rule != nil && rule.ProcessingStrategy != "" { - value, ok := protos.ProcessingStrategy_value[rule.ProcessingStrategy] + if options.rule != nil && options.rule.ProcessingStrategy != "" { + value, ok := protos.ProcessingStrategy_value[options.rule.ProcessingStrategy] if !ok { span.SetAttributes( attribute.String("health_events_analyzer.error.type", "invalid_processing_strategy"), attribute.String("health_events_analyzer.error.message", - fmt.Sprintf("unexpected processingStrategy: %q", rule.ProcessingStrategy)), + fmt.Sprintf("unexpected processingStrategy: %q", options.rule.ProcessingStrategy)), ) - tracing.RecordError(span, fmt.Errorf("unexpected processingStrategy value: %q", rule.ProcessingStrategy)) + tracing.RecordError(span, fmt.Errorf("unexpected processingStrategy value: %q", options.rule.ProcessingStrategy)) - return fmt.Errorf("unexpected processingStrategy value: %q", rule.ProcessingStrategy) + return client.PermanentError( + fmt.Errorf("unexpected processingStrategy value: %q", options.rule.ProcessingStrategy), + ) } newEvent.ProcessingStrategy = protos.ProcessingStrategy(value) } - if recommendedAction == protos.RecommendedAction_NONE { + switch { + case options.isHealthy: newEvent.IsFatal = false - } else { + newEvent.ErrorCode = nil + newEvent.EntitiesImpacted = cloneEntities(options.entities) + newEvent.QuarantineOverrides = nil + newEvent.DrainOverrides = nil + newEvent.CustomRecommendedAction = "" + case options.recommendedAction == protos.RecommendedAction_NONE: + newEvent.IsFatal = false + default: newEvent.IsFatal = true } @@ -184,3 +234,20 @@ func (p *PublisherConfig) Publish(ctx context.Context, event *protos.HealthEvent return p.sendHealthEventWithRetry(ctx, req) } + +func cloneEntities(entities []*protos.Entity) []*protos.Entity { + if len(entities) == 0 { + return nil + } + + clones := make([]*protos.Entity, 0, len(entities)) + for _, entity := range entities { + if entity == nil { + continue + } + + clones = append(clones, proto.Clone(entity).(*protos.Entity)) + } + + return clones +} diff --git a/health-events-analyzer/pkg/publisher/publisher_test.go b/health-events-analyzer/pkg/publisher/publisher_test.go index 36e78c9d6..bec457914 100644 --- a/health-events-analyzer/pkg/publisher/publisher_test.go +++ b/health-events-analyzer/pkg/publisher/publisher_test.go @@ -16,17 +16,171 @@ package publisher import ( "context" + "errors" "testing" "time" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/emptypb" "google.golang.org/protobuf/types/known/timestamppb" - protos "github.com/nvidia/nvsentinel/data-models/pkg/protos" + "github.com/nvidia/nvsentinel/data-models/pkg/protos" + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/config" + storeclient "github.com/nvidia/nvsentinel/store-client/pkg/client" ) +type capturePlatformConnector struct { + events *protos.HealthEvents +} + +type unavailablePlatformConnector struct{} + +func (*unavailablePlatformConnector) HealthEventOccurredV1( + context.Context, + *protos.HealthEvents, + ...grpc.CallOption, +) (*emptypb.Empty, error) { + return nil, status.Error(codes.Unavailable, "connector unavailable") +} + +func (c *capturePlatformConnector) HealthEventOccurredV1( + _ context.Context, + events *protos.HealthEvents, + _ ...grpc.CallOption, +) (*emptypb.Empty, error) { + c.events = proto.Clone(events).(*protos.HealthEvents) + return &emptypb.Empty{}, nil +} + +func TestPublishRecovery(t *testing.T) { + client := &capturePlatformConnector{} + pub := NewPublisher(client, protos.ProcessingStrategy_EXECUTE_REMEDIATION) + sourceGeneratedTime := time.Date(2026, 8, 21, 8, 27, 36, 0, time.UTC) + source := &protos.HealthEvent{ + Version: 1, + Agent: "syslog-health-monitor", + ComponentClass: "GPU", + CheckName: "SysLogsXIDError", + IsFatal: true, + IsHealthy: true, + RecommendedAction: protos.RecommendedAction_RESTART_BM, + CustomRecommendedAction: "old-custom-action", + ErrorCode: []string{"94"}, + EntitiesImpacted: []*protos.Entity{ + {EntityType: "GPU", EntityValue: "0"}, + {EntityType: "GPU_UUID", EntityValue: "GPU-123"}, + }, + Metadata: map[string]string{"source": "reboot-check"}, + GeneratedTimestamp: timestamppb.New(sourceGeneratedTime), + NodeName: "node-a", + QuarantineOverrides: &protos.BehaviourOverrides{Force: true}, + DrainOverrides: &protos.BehaviourOverrides{Skip: true}, + ProcessingStrategy: protos.ProcessingStrategy_STORE_AND_ANALYSE, + } + original := proto.Clone(source).(*protos.HealthEvent) + rule := &config.HealthEventsAnalyzerRule{ + Name: "RepeatedXID94OnSameGPU", + ProcessingStrategy: "EXECUTE_REMEDIATION", + } + before := time.Now() + + err := pub.PublishRecovery(context.Background(), source, rule.Name, + []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-123"}}, rule) + require.NoError(t, err) + require.True(t, proto.Equal(original, source), "publishing must not mutate the source event") + require.NotNil(t, client.events) + require.Len(t, client.events.Events, 1) + + recovery := client.events.Events[0] + require.Equal(t, "health-events-analyzer", recovery.Agent) + require.Equal(t, rule.Name, recovery.CheckName) + require.True(t, recovery.IsHealthy) + require.False(t, recovery.IsFatal) + require.Equal(t, protos.RecommendedAction_NONE, recovery.RecommendedAction) + require.Empty(t, recovery.CustomRecommendedAction) + require.Empty(t, recovery.ErrorCode) + require.Equal(t, []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-123"}}, recovery.EntitiesImpacted) + require.Nil(t, recovery.QuarantineOverrides) + require.Nil(t, recovery.DrainOverrides) + require.Equal(t, protos.ProcessingStrategy_EXECUTE_REMEDIATION, recovery.ProcessingStrategy) + require.Equal(t, "reboot-check", recovery.Metadata["source"]) + require.Equal(t, sourceGeneratedTime.Format(time.RFC3339Nano), + recovery.Metadata[SourceGeneratedTimestampMetadataKey]) + require.NotNil(t, recovery.GeneratedTimestamp) + require.NotEqual(t, sourceGeneratedTime, recovery.GeneratedTimestamp.AsTime()) + require.False(t, recovery.GeneratedTimestamp.AsTime().Before(before.Add(-time.Second))) + require.Equal(t, source.NodeName, recovery.NodeName) +} + +func TestPublishRecoveryNodeScopeHasNoEntities(t *testing.T) { + client := &capturePlatformConnector{} + pub := NewPublisher(client, protos.ProcessingStrategy_EXECUTE_REMEDIATION) + + err := pub.PublishRecovery(context.Background(), &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{ + {EntityType: "GPU_UUID", EntityValue: "GPU-123"}, + }, + }, "NodeDerivedCondition", nil, nil) + require.NoError(t, err) + require.Empty(t, client.events.Events[0].EntitiesImpacted) +} + +func TestPublishRecoveryRejectsInvalidProcessingStrategy(t *testing.T) { + client := &capturePlatformConnector{} + pub := NewPublisher(client, protos.ProcessingStrategy_EXECUTE_REMEDIATION) + rule := &config.HealthEventsAnalyzerRule{ProcessingStrategy: "NOT_A_STRATEGY"} + + err := pub.PublishRecovery(context.Background(), &protos.HealthEvent{NodeName: "node-a"}, + "DerivedCondition", nil, rule) + require.ErrorContains(t, err, "unexpected processingStrategy value") + require.True(t, storeclient.IsPermanentError(err)) + require.Nil(t, client.events) +} + +func TestPublishPreservesUnhealthyEventSemantics(t *testing.T) { + client := &capturePlatformConnector{} + pub := NewPublisher(client, protos.ProcessingStrategy_EXECUTE_REMEDIATION) + source := &protos.HealthEvent{ + Agent: "source-monitor", + CheckName: "SourceCheck", + IsHealthy: false, + IsFatal: false, + EntitiesImpacted: []*protos.Entity{nil, {EntityType: "GPU_UUID", EntityValue: "GPU-1"}}, + } + + err := pub.Publish(context.Background(), source, protos.RecommendedAction_NONE, + "DerivedCondition", "derived", nil) + require.NoError(t, err) + require.NotNil(t, client.events) + derived := client.events.Events[0] + require.Equal(t, "health-events-analyzer", derived.Agent) + require.Equal(t, "DerivedCondition", derived.CheckName) + require.False(t, derived.IsHealthy) + require.False(t, derived.IsFatal) + + clones := cloneEntities(source.EntitiesImpacted) + require.Len(t, clones, 1) + require.True(t, proto.Equal( + &protos.Entity{EntityType: "GPU_UUID", EntityValue: "GPU-1"}, clones[0], + )) +} + +func TestPublishRetryHonorsContextDeadline(t *testing.T) { + pub := NewPublisher(&unavailablePlatformConnector{}, protos.ProcessingStrategy_EXECUTE_REMEDIATION) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + err := pub.Publish(ctx, &protos.HealthEvent{}, protos.RecommendedAction_NONE, + "DerivedCondition", "derived", nil) + require.Error(t, err) + require.True(t, errors.Is(err, context.DeadlineExceeded), err) +} + type fakePlatformConnectorClient struct { events *protos.HealthEvents } @@ -77,7 +231,7 @@ func TestPublish_LaggingSourceEvent_StampsPublishTimeAndKeepsSourceTimestamp(t * // The source timestamp is preserved so provenance is not lost. require.Equal(t, sourceTime.Format(time.RFC3339Nano), - published.GetMetadata()[sourceGeneratedTimestampMetadataKey]) + published.GetMetadata()[SourceGeneratedTimestampMetadataKey]) } // Asserts the wire key literally rather than through the constant, so a rename cannot @@ -112,7 +266,7 @@ func TestPublish_SourceWithMetadata_PreservesExistingKeys(t *testing.T) { published := client.events.GetEvents()[0] require.Equal(t, "value", published.GetMetadata()["existing"]) require.Equal(t, sourceTime.Format(time.RFC3339Nano), - published.GetMetadata()[sourceGeneratedTimestampMetadataKey]) + published.GetMetadata()[SourceGeneratedTimestampMetadataKey]) } func TestPublish_SourceWithoutTimestamp_StampsWithoutSourceMetadata(t *testing.T) { @@ -128,7 +282,7 @@ func TestPublish_SourceWithoutTimestamp_StampsWithoutSourceMetadata(t *testing.T published := client.events.GetEvents()[0] require.NotNil(t, published.GetGeneratedTimestamp()) - require.NotContains(t, published.GetMetadata(), sourceGeneratedTimestampMetadataKey) + require.NotContains(t, published.GetMetadata(), SourceGeneratedTimestampMetadataKey) } func TestPublish_AnySourceEvent_DoesNotMutateCaller(t *testing.T) { @@ -145,5 +299,5 @@ func TestPublish_AnySourceEvent_DoesNotMutateCaller(t *testing.T) { // Publish clones, so the caller's event must be untouched. require.True(t, src.GetGeneratedTimestamp().AsTime().Equal(sourceTime)) require.Equal(t, "syslog-health-monitor", src.GetAgent()) - require.NotContains(t, src.GetMetadata(), sourceGeneratedTimestampMetadataKey) + require.NotContains(t, src.GetMetadata(), SourceGeneratedTimestampMetadataKey) } diff --git a/health-events-analyzer/pkg/reconciler/metrics.go b/health-events-analyzer/pkg/reconciler/metrics.go index 1a3d802b4..6983f5db5 100644 --- a/health-events-analyzer/pkg/reconciler/metrics.go +++ b/health-events-analyzer/pkg/reconciler/metrics.go @@ -19,13 +19,18 @@ import ( "github.com/prometheus/client_golang/prometheus/promauto" ) +const ( + metricLabelNodeName = "node_name" + metricLabelRuleName = "rule_name" +) + var ( totalEventsReceived = promauto.NewCounterVec( prometheus.CounterOpts{ Name: "health_event_analyzer_events_received_total", Help: "Total number of events received from the watcher.", }, - []string{"node_name"}, + []string{metricLabelNodeName}, ) totalEventsSuccessfullyProcessed = promauto.NewCounter( prometheus.CounterOpts{ @@ -54,7 +59,29 @@ var ( Name: "rule_matched_total", Help: "Total number of times a rule matched for a node", }, - []string{"rule_name", "node_name"}, + []string{metricLabelRuleName, metricLabelNodeName}, + ) + + recoveryEventsPublishedTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "health_event_analyzer_recovery_events_published_total", + Help: "Total number of derived healthy transitions published by recovery-enabled rules.", + }, + []string{metricLabelRuleName, "scope"}, + ) + recoveryPersistenceTimeoutsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "health_event_analyzer_recovery_persistence_timeouts_total", + Help: "Total derived transitions not observed in the store before the persistence deadline.", + }, + []string{metricLabelRuleName, "state"}, + ) + recoveryStoredDocumentDecodeErrorsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "health_event_analyzer_recovery_stored_document_decode_errors_total", + Help: "Total stored health event documents skipped because they could not be decoded or scoped.", + }, + []string{metricLabelRuleName, "lookup", "classification"}, ) mongoQueryExecutionDuration = promauto.NewHistogramVec( @@ -63,7 +90,7 @@ var ( Help: "Histogram of MongoDB pipeline execution durations.", Buckets: prometheus.DefBuckets, }, - []string{"rule_name"}, + []string{metricLabelRuleName}, ) // performance metrics @@ -71,7 +98,7 @@ var ( prometheus.HistogramOpts{ Name: "health_event_analyzer_event_handling_duration_seconds", Help: "Histogram of event handling durations.", - Buckets: prometheus.DefBuckets, + Buckets: prometheus.ExponentialBuckets(0.1, 2, 12), }, ) ) diff --git a/health-events-analyzer/pkg/reconciler/reconciler.go b/health-events-analyzer/pkg/reconciler/reconciler.go index 32a50f3c1..ac49b3d83 100644 --- a/health-events-analyzer/pkg/reconciler/reconciler.go +++ b/health-events-analyzer/pkg/reconciler/reconciler.go @@ -18,11 +18,13 @@ import ( "context" "fmt" "log/slog" + "sync" "time" multierror "github.com/hashicorp/go-multierror" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/nvidia/nvsentinel/commons/pkg/healthstatus" "github.com/nvidia/nvsentinel/commons/pkg/tracing" @@ -48,6 +50,13 @@ const ( // fieldNodeName is the stored document field used to scope rule evaluation // to the node that produced the incoming event. fieldNodeName = "healthevent.nodename" + // fieldGeneratedTimestamp is used to preserve created-at fallback semantics + // for legacy events without a generated timestamp. + fieldGeneratedTimestamp = "healthevent.generatedtimestamp" + // aggregationOperatorGT is the greater-than operator used in generated + // analyzer aggregation stages. + aggregationOperatorGT = "$gt" + aggregationOperatorOr = "$or" ) type HealthEventsAnalyzerReconcilerConfig struct { @@ -62,8 +71,17 @@ type Reconciler struct { datastore datastore.DataStore databaseClient client.DatabaseClient // MongoDB-specific client for aggregation eventProcessor client.EventProcessor - xidDetector *analyzer.XidBurstDetector // PostgreSQL-specific XID burst detection - useXidDetector bool // True if using PostgreSQL + // XID burst detection; enabled only on the PostgreSQL provider (see Start). + xidDetector *analyzer.XidBurstDetector + useXidDetector bool + provider datastore.DataStoreProvider + recoveryMu sync.RWMutex + recoveryBoundaries map[string]recoveryBoundary + recoveryLoaded map[string]struct{} + derivedStates map[string]derivedState + recoveryPoll time.Duration + recoveryRepublish time.Duration + recoveryTimeout time.Duration } func NewReconciler(cfg HealthEventsAnalyzerReconcilerConfig) *Reconciler { @@ -83,6 +101,7 @@ func (r *Reconciler) Start(ctx context.Context) error { defer ds.Close(ctx) r.datastore = ds + r.provider = ds.Provider() // Check if using PostgreSQL and enable XID burst detector if ds.Provider() == datastore.ProviderPostgreSQL { @@ -129,9 +148,8 @@ func (r *Reconciler) Start(ctx context.Context) error { oldWatcher := unwrapable.Unwrap() - // Create and configure the unified EventProcessor - // Note: EventProcessor no longer retries internally to prevent blocking the event stream - // Failed events will be retried on next pod restart (via resume token) + // The handler owns bounded retries. If an event remains uncheckpointed, the + // processor stops so a later event cannot advance the resume token past it. processorConfig := client.EventProcessorConfig{ EnableMetrics: true, MetricsLabels: map[string]string{"module": agentName}, @@ -201,18 +219,7 @@ func (r *Reconciler) processHealthEvent(ctx context.Context, event *datamodels.H attribute.Bool("health_events_analyzer.event.published", publishedNewEvent), ) - if publishedNewEvent { - slog.InfoContext(ctx, "New fatal event published.") - // Only track entity-specific metrics if EntitiesImpacted is not empty - if len(event.HealthEvent.EntitiesImpacted) > 0 { - fatalEventsPublishedTotal.WithLabelValues(event.HealthEvent.EntitiesImpacted[0].EntityValue).Inc() - } else { - slog.WarnContext(ctx, "Fatal event published but EntitiesImpacted is empty, using 'unknown' for metrics") - fatalEventsPublishedTotal.WithLabelValues("unknown").Inc() - } - } else { - slog.InfoContext(ctx, "Fatal event is not published, rule set criteria didn't match.") - } + r.recordPublishedEvent(ctx, event.HealthEvent, publishedNewEvent) // Track processing duration duration := time.Since(startTime).Seconds() @@ -221,6 +228,31 @@ func (r *Reconciler) processHealthEvent(ctx context.Context, event *datamodels.H return nil } +func (r *Reconciler) recordPublishedEvent(ctx context.Context, event *protos.HealthEvent, published bool) { + if !published { + slog.InfoContext(ctx, "No derived event published.") + + return + } + + if event.IsHealthy { + slog.InfoContext(ctx, "Derived recovery event published.") + + return + } + + slog.InfoContext(ctx, "New fatal event published.") + + if len(event.EntitiesImpacted) == 0 { + slog.WarnContext(ctx, "Fatal event published but EntitiesImpacted is empty, using 'unknown' for metrics") + fatalEventsPublishedTotal.WithLabelValues("unknown").Inc() + + return + } + + fatalEventsPublishedTotal.WithLabelValues(event.EntitiesImpacted[0].EntityValue).Inc() +} + func (r *Reconciler) handleEvent(ctx context.Context, event *datamodels.HealthEventWithStatus) (bool, error) { ctx, span := tracing.StartSpan(ctx, "health_events_analyzer.handle_event") defer span.End() @@ -229,25 +261,81 @@ func (r *Reconciler) handleEvent(ctx context.Context, event *datamodels.HealthEv publishedNewEvent := false - // Handle XID detector operations (clear on healthy, detect bursts on unhealthy) - published, err := r.handleXidDetector(ctx, event) + // Healthy events are admitted only for configured recovery mappings. Keep + // the XID detector on its existing unhealthy-event input. + if !event.HealthEvent.IsHealthy { + published, err := r.handleXidDetector(ctx, event) + if err != nil { + multiErr = multierror.Append(multiErr, err) + } + + if published { + publishedNewEvent = true + } + } + + published, err := r.processHealthState(ctx, event, span) if err != nil { multiErr = multierror.Append(multiErr, err) } - if published { - publishedNewEvent = true + publishedNewEvent = published || publishedNewEvent + + if multiErr.ErrorOrNil() != nil { + slog.ErrorContext(ctx, "Error in handling the event", "error", multiErr) + span.SetAttributes( + attribute.String("health_events_analyzer.error.type", "handle_event_error"), + attribute.String("health_events_analyzer.error.message", multiErr.Error()), + ) + tracing.RecordError(span, multiErr.ErrorOrNil()) + + return publishedNewEvent, fmt.Errorf("error in handling the event: %w", multiErr) + } + + return publishedNewEvent, nil +} + +func (r *Reconciler) processHealthState( + ctx context.Context, + event *datamodels.HealthEventWithStatus, + span trace.Span, +) (bool, error) { + if event.HealthEvent.IsHealthy { + return r.handleRecoveryEvents(ctx, event) } - // Process regular rules + return r.processConfiguredRules(ctx, event, span) +} + +func (r *Reconciler) processConfiguredRules( + ctx context.Context, + event *datamodels.HealthEventWithStatus, + span trace.Span, +) (bool, error) { + var multiErr *multierror.Error + + publishedAny := false + for _, rule := range r.config.HealthEventsAnalyzerRules.Rules { if !rule.EvaluateRule { slog.InfoContext(ctx, "Skipping rule evaluation", "rule_name", rule.Name) + continue } published, err := r.processRule(ctx, rule, event) if err != nil { + if client.IsPermanentError(err) { + slog.ErrorContext(ctx, "Skipping rule after deterministic evaluation failure", + "rule_name", rule.Name, "error", err) + totalEventProcessingError.WithLabelValues("permanent_rule_error").Inc() + span.AddEvent("permanent_rule_error", trace.WithAttributes( + attribute.String("health_events_analyzer.error.message", err.Error()), + )) + + continue + } + multiErr = multierror.Append(multiErr, err) span.AddEvent("rule_evaluation_error", trace.WithAttributes( attribute.String("health_events_analyzer.error.type", "rule_evaluation_error"), @@ -257,23 +345,10 @@ func (r *Reconciler) handleEvent(ctx context.Context, event *datamodels.HealthEv continue } - if published { - publishedNewEvent = true - } - } - - if multiErr.ErrorOrNil() != nil { - slog.ErrorContext(ctx, "Error in handling the event", "error", multiErr) - span.SetAttributes( - attribute.String("health_events_analyzer.error.type", "handle_event_error"), - attribute.String("health_events_analyzer.error.message", multiErr.Error()), - ) - tracing.RecordError(span, multiErr.ErrorOrNil()) - - return publishedNewEvent, fmt.Errorf("error in handling the event: %w", multiErr) + publishedAny = published || publishedAny } - return publishedNewEvent, nil + return publishedAny, multiErr.ErrorOrNil() } // handleXidDetector handles XID burst detection and history clearing @@ -285,14 +360,6 @@ func (r *Reconciler) handleXidDetector(ctx context.Context, event *datamodels.He ctx, span := tracing.StartSpan(ctx, "health_events_analyzer.handle_xid_detector") defer span.End() - // Clear XID burst history when a healthy GPU event is received - if r.shouldClearXidHistory(event.HealthEvent) { - r.xidDetector.ClearNodeHistory(event.HealthEvent.NodeName) - span.SetAttributes(attribute.Bool("health_events_analyzer.xid.history_cleared", true)) - slog.InfoContext(ctx, "Cleared XID burst history for node due to healthy GPU event", - "node", event.HealthEvent.NodeName) - } - // Check for GPU XID errors and detect burst patterns if r.shouldProcessXidEvent(event.HealthEvent) { published, err := r.processXidBurstDetection(ctx, event.HealthEvent) @@ -353,7 +420,19 @@ func (r *Reconciler) processRule(ctx context.Context, return false, nil } - err = r.publishMatchedEvent(ctx, rule, event) + identity, recoveryEnabled := recoveryIdentityForEvent(rule, event.HealthEvent) + + if rule.Recovery != nil && !recoveryEnabled { + slog.WarnContext(ctx, "Rule match does not contain the configured recovery scope; "+ + "publishing without automatic recovery", + "rule_name", rule.Name, + "node", event.HealthEvent.NodeName, + "entity_types", rule.Recovery.EntityTypes) + } + + ruleMatchedTotal.WithLabelValues(rule.Name, event.HealthEvent.NodeName).Inc() + + published, err := r.publishRuleMatch(ctx, rule, event, identity, recoveryEnabled) if err != nil { slog.ErrorContext(ctx, "Error in publishing the matched event", "error", err) span.SetAttributes( @@ -365,21 +444,47 @@ func (r *Reconciler) processRule(ctx context.Context, return false, fmt.Errorf("error in publishing the matched event: %w", err) } - return true, nil + return published, nil +} + +func (r *Reconciler) publishRuleMatch( + ctx context.Context, + rule config.HealthEventsAnalyzerRule, + event *datamodels.HealthEventWithStatus, + identity recoveryIdentity, + recoveryEnabled bool, +) (bool, error) { + if !recoveryEnabled { + if err := r.publishMatchedEvent(ctx, rule, event.HealthEvent); err != nil { + return false, err + } + + return true, nil + } + + persistedBoundary, published, err := r.publishFaultUntilStored(ctx, event, rule, identity) + if err != nil { + return false, err + } + + r.rememberDerivedState(rule.Name, identity, derivedState{ + boundary: persistedBoundary, + isHealthy: false, + }) + + return published, nil } // publishMatchedEvent publishes an event when a rule matches func (r *Reconciler) publishMatchedEvent(ctx context.Context, rule config.HealthEventsAnalyzerRule, - event *datamodels.HealthEventWithStatus) error { + event *protos.HealthEvent) error { ctx, span := tracing.StartSpan(ctx, "health_events_analyzer.publish_matched_event") defer span.End() - ruleMatchedTotal.WithLabelValues(rule.Name, event.HealthEvent.NodeName).Inc() - actionVal := r.getRecommendedActionValue(rule.RecommendedAction, rule.Name) - err := r.config.Publisher.Publish(ctx, event.HealthEvent, protos.RecommendedAction(actionVal), + err := r.config.Publisher.Publish(ctx, event, protos.RecommendedAction(actionVal), rule.Name, rule.Message, &rule) if err != nil { slog.ErrorContext(ctx, "Error in publishing the new fatal event", "error", err) @@ -395,7 +500,7 @@ func (r *Reconciler) publishMatchedEvent(ctx context.Context, } slog.InfoContext(ctx, "New event successfully published for matching rule", - "rule_name", rule.Name, "node", event.HealthEvent.NodeName) + "rule_name", rule.Name, "node", event.NodeName) return nil } @@ -428,8 +533,13 @@ func (r *Reconciler) validateAllSequenceCriteria(ctx context.Context, rule confi "error_code", healthEventWithStatus.HealthEvent.ErrorCode, "agent", healthEventWithStatus.HealthEvent.Agent) + boundary, err := r.recoveryBoundaryForEvent(ctx, rule, healthEventWithStatus.HealthEvent) + if err != nil { + return false, fmt.Errorf("find recovery boundary: %w", err) + } + // Build aggregation pipeline from stages - pipelineStages, err := r.getPipelineStages(rule, healthEventWithStatus) + pipelineStages, err := r.getPipelineStages(rule, healthEventWithStatus, boundary) if err != nil { slog.ErrorContext(ctx, "Failed to build pipeline stages", "error", err) tracing.RecordError(span, err) @@ -446,7 +556,12 @@ func (r *Reconciler) validateAllSequenceCriteria(ctx context.Context, rule confi slog.DebugContext(ctx, "Executing aggregation pipeline", "rule_name", rule.Name, "pipeline_stages_count", len(pipelineStages)) - cursor, err := r.databaseClient.Aggregate(ctx, pipelineStages) + queryPipeline := any(client.WithExtendedFilterPrefix(pipelineStages, 1)) + if rule.Recovery != nil { + queryPipeline = client.WithExtendedFilters(pipelineStages) + } + + cursor, err := r.databaseClient.Aggregate(ctx, queryPipeline) if err != nil { slog.ErrorContext(ctx, "Failed to execute aggregation pipeline", "error", err, "rule_name", rule.Name) totalEventProcessingError.WithLabelValues("execute_pipeline_error").Inc() @@ -457,7 +572,9 @@ func (r *Reconciler) validateAllSequenceCriteria(ctx context.Context, rule confi ) tracing.RecordError(span, err) - return false, fmt.Errorf("failed to execute aggregation pipeline: %w", err) + return false, classifyRuleDatastoreError( + fmt.Errorf("failed to execute aggregation pipeline: %w", err), + ) } defer cursor.Close(ctx) @@ -472,7 +589,7 @@ func (r *Reconciler) validateAllSequenceCriteria(ctx context.Context, rule confi ) tracing.RecordError(span, err) - return false, fmt.Errorf("failed to decode cursor: %w", err) + return false, classifyRuleDatastoreError(fmt.Errorf("failed to decode cursor: %w", err)) } slog.DebugContext(ctx, "Aggregation pipeline completed", "rule_name", rule.Name, "result_count", len(result)) @@ -511,29 +628,56 @@ func (r *Reconciler) validateAllSequenceCriteria(ctx context.Context, rule confi func (r *Reconciler) getPipelineStages( rule config.HealthEventsAnalyzerRule, healthEventWithStatus datamodels.HealthEventWithStatus, + boundary *recoveryBoundary, ) ([]map[string]any, error) { // Always start with mandatory filters. The agent filter prevents the analyzer // from matching its own generated events, while the node filter limits each // rule evaluation to events from the node that produced the incoming event. // Keeping the node predicate in the first stage lets the datastore use its // node-prefixed HealthEvents index before evaluating configured rule stages. - pipeline := []map[string]any{ - { - "$match": map[string]any{ - "healthevent.agent": map[string]any{"$ne": agentName}, - fieldNodeName: healthEventWithStatus.HealthEvent.NodeName, - "$or": []any{ - map[string]any{ - fieldProcessingStrategy: int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), - }, - map[string]any{ - fieldProcessingStrategy: int32(protos.ProcessingStrategy_STORE_AND_ANALYSE), - }, - map[string]any{ - fieldProcessingStrategy: map[string]any{"$exists": false}, + mandatoryMatch := map[string]any{ + "healthevent.agent": map[string]any{"$ne": agentName}, + fieldNodeName: healthEventWithStatus.HealthEvent.NodeName, + aggregationOperatorOr: []any{ + map[string]any{ + fieldProcessingStrategy: int32(protos.ProcessingStrategy_UNSPECIFIED), + }, + map[string]any{ + fieldProcessingStrategy: int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), + }, + map[string]any{ + fieldProcessingStrategy: int32(protos.ProcessingStrategy_STORE_AND_ANALYSE), + }, + map[string]any{ + fieldProcessingStrategy: map[string]any{"$exists": false}, + }, + }, + } + + if boundary != nil { + if !boundary.createdAt.IsZero() { + mandatoryMatch["createdAt"] = map[string]any{aggregationOperatorGT: boundary.createdAt} + } + + if boundary.generated != nil { + mandatoryMatch["$and"] = []any{ + map[string]any{ + aggregationOperatorOr: []any{ + map[string]any{ + fieldGeneratedTimestamp: map[string]any{"$exists": false}, + }, + map[string]any{ + "$expr": generatedAfterExpression(boundary.generated), + }, }, }, - }, + } + } + } + + pipeline := []map[string]any{ + { + "$match": mandatoryMatch, }, } @@ -544,7 +688,7 @@ func (r *Reconciler) getPipelineStages( slog.Error("Failed to parse stage", "stage_index", i, "error", err, "stage_string", stageStr) totalEventProcessingError.WithLabelValues("parse_stage_error").Inc() - return nil, fmt.Errorf("failed to parse stage %d: %w", i, err) + return nil, client.PermanentError(fmt.Errorf("failed to parse stage %d: %w", i, err)) } slog.Debug("Parsed aggregation stage", "rule_name", rule.Name, "stage_index", i) @@ -555,6 +699,34 @@ func (r *Reconciler) getPipelineStages( return pipeline, nil } +func classifyRuleDatastoreError(err error) error { + if datastore.IsDeterministicError(err) { + return client.PermanentError(err) + } + + return err +} + +func generatedAfterExpression(timestamp *timestamppb.Timestamp) map[string]any { + return map[string]any{ + aggregationOperatorOr: []any{ + map[string]any{ + aggregationOperatorGT: []any{"$healthevent.generatedtimestamp.seconds", timestamp.Seconds}, + }, + map[string]any{ + "$and": []any{ + map[string]any{ + "$eq": []any{"$healthevent.generatedtimestamp.seconds", timestamp.Seconds}, + }, + map[string]any{ + aggregationOperatorGT: []any{"$healthevent.generatedtimestamp.nanos", int64(timestamp.Nanos)}, + }, + }, + }, + }, + } +} + // shouldProcessXidEvent checks if an event should be processed by the XID burst detector func (r *Reconciler) shouldProcessXidEvent(event *protos.HealthEvent) bool { // Only process GPU XID errors (unhealthy GPU events with error codes) @@ -565,16 +737,6 @@ func (r *Reconciler) shouldProcessXidEvent(event *protos.HealthEvent) bool { event.Agent != agentName // Don't process our own events } -// shouldClearXidHistory checks if a healthy GPU event should clear the XID burst history -// This ensures that when a GPU is healthy again, we don't keep triggering RepeatedXidError -// based on stale XID history from before the recovery -func (r *Reconciler) shouldClearXidHistory(event *protos.HealthEvent) bool { - return event != nil && - event.ComponentClass == "GPU" && - event.IsHealthy && - event.Agent != agentName // Don't process our own events -} - // processXidBurstDetection processes GPU XID events through the burst detector // and publishes RepeatedXidError events when burst patterns are detected func (r *Reconciler) processXidBurstDetection(ctx context.Context, event *protos.HealthEvent) (bool, error) { diff --git a/health-events-analyzer/pkg/reconciler/reconciler_agent_filter_test.go b/health-events-analyzer/pkg/reconciler/reconciler_agent_filter_test.go index 6adda6e77..e7cb0a81f 100644 --- a/health-events-analyzer/pkg/reconciler/reconciler_agent_filter_test.go +++ b/health-events-analyzer/pkg/reconciler/reconciler_agent_filter_test.go @@ -112,7 +112,7 @@ func TestGetPipelineStages_AlwaysIncludesAgentFilter(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - pipeline, err := reconciler.getPipelineStages(tc.rule, tc.event) + pipeline, err := reconciler.getPipelineStages(tc.rule, tc.event, nil) require.NoError(t, err, "getPipelineStages should not return an error") // Verify pipeline has at least the agent filter stage @@ -163,7 +163,7 @@ func TestGetPipelineStages_AlwaysIncludesNodeFilter(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) require.NoError(t, err) require.Len(t, pipeline, 3) @@ -199,7 +199,7 @@ func TestGetPipelineStages_AgentFilterPreventsInfiniteLoop(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, eventFromAnalyzer) + pipeline, err := reconciler.getPipelineStages(rule, eventFromAnalyzer, nil) require.NoError(t, err) // Extract the agent filter from the first stage @@ -240,7 +240,7 @@ func TestGetPipelineStages_AgentFilterPosition(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) require.NoError(t, err) // Pipeline should have: 1 agent filter + 3 configured stages = 4 total diff --git a/health-events-analyzer/pkg/reconciler/reconciler_test.go b/health-events-analyzer/pkg/reconciler/reconciler_test.go index 32261716b..c763c2125 100644 --- a/health-events-analyzer/pkg/reconciler/reconciler_test.go +++ b/health-events-analyzer/pkg/reconciler/reconciler_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "google.golang.org/grpc" @@ -31,6 +32,7 @@ import ( config "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/config" "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/publisher" "github.com/nvidia/nvsentinel/store-client/pkg/client" + "github.com/nvidia/nvsentinel/store-client/pkg/datastore" ) // Mock Publisher @@ -38,6 +40,128 @@ type mockPublisher struct { mock.Mock } +type startTestClientWatcher struct { + events chan client.Event + started chan struct{} +} + +func (w *startTestClientWatcher) Start(ctx context.Context) { + close(w.started) + <-ctx.Done() +} + +func (w *startTestClientWatcher) Events() <-chan client.Event { return w.events } +func (w *startTestClientWatcher) MarkProcessed(context.Context, []byte) error { + return nil +} +func (w *startTestClientWatcher) Close(context.Context) error { return nil } + +type startTestWatcher struct { + inner *startTestClientWatcher + events chan datastore.EventWithToken +} + +func (w *startTestWatcher) Events() <-chan datastore.EventWithToken { return w.events } +func (w *startTestWatcher) Start(ctx context.Context) { w.inner.Start(ctx) } +func (w *startTestWatcher) MarkProcessed(context.Context, []byte) error { + return nil +} +func (w *startTestWatcher) Close(context.Context) error { return nil } +func (w *startTestWatcher) Unwrap() client.ChangeStreamWatcher { return w.inner } + +type startTestDataStore struct { + provider datastore.DataStoreProvider + database client.DatabaseClient + watcher datastore.ChangeStreamWatcher +} + +func (s *startTestDataStore) MaintenanceEventStore() datastore.MaintenanceEventStore { return nil } +func (s *startTestDataStore) HealthEventStore() datastore.HealthEventStore { return nil } +func (s *startTestDataStore) Ping(context.Context) error { return nil } +func (s *startTestDataStore) Close(context.Context) error { return nil } +func (s *startTestDataStore) Provider() datastore.DataStoreProvider { return s.provider } +func (s *startTestDataStore) GetDatabaseClient() client.DatabaseClient { return s.database } +func (s *startTestDataStore) CreateChangeStreamWatcher( + context.Context, string, any, +) (datastore.ChangeStreamWatcher, error) { + return s.watcher, nil +} + +func TestStartWiresProviderAndProcessor(t *testing.T) { + provider := datastore.DataStoreProvider("health-events-analyzer-start-test") + inner := &startTestClientWatcher{ + events: make(chan client.Event), + started: make(chan struct{}), + } + store := &startTestDataStore{ + provider: provider, + database: new(mockDatabaseClient), + watcher: &startTestWatcher{ + inner: inner, events: make(chan datastore.EventWithToken), + }, + } + datastore.RegisterProvider(provider, func(context.Context, datastore.DataStoreConfig) (datastore.DataStore, error) { + return store, nil + }) + + reconciler := NewReconciler(HealthEventsAnalyzerReconcilerConfig{ + DataStoreConfig: &datastore.DataStoreConfig{Provider: provider}, + HealthEventsAnalyzerRules: &config.TomlConfig{}, + }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- reconciler.Start(ctx) }() + <-inner.started + cancel() + + assert.ErrorIs(t, <-done, context.Canceled) + assert.Equal(t, provider, reconciler.provider) + assert.NotNil(t, reconciler.eventProcessor) +} + +func TestRecordPublishedEventMetrics(t *testing.T) { + reconciler := &Reconciler{} + ctx := context.Background() + + reconciler.recordPublishedEvent(ctx, &protos.HealthEvent{}, false) + reconciler.recordPublishedEvent(ctx, &protos.HealthEvent{IsHealthy: true}, true) + + fatalEventsPublishedTotal.WithLabelValues("unknown") + unknownBefore := fatalEventCounterValue(t, "unknown") + reconciler.recordPublishedEvent(ctx, &protos.HealthEvent{}, true) + assert.Equal(t, unknownBefore+1, fatalEventCounterValue(t, "unknown")) + + entity := "GPU-record-published-test" + fatalEventsPublishedTotal.WithLabelValues(entity) + entityBefore := fatalEventCounterValue(t, entity) + reconciler.recordPublishedEvent(ctx, &protos.HealthEvent{ + EntitiesImpacted: []*protos.Entity{{EntityValue: entity}}, + }, true) + assert.Equal(t, entityBefore+1, fatalEventCounterValue(t, entity)) +} + +func fatalEventCounterValue(t *testing.T, entity string) float64 { + t.Helper() + + families, err := prometheus.DefaultGatherer.Gather() + assert.NoError(t, err) + for _, family := range families { + if family.GetName() != "fatal_events_published_total" { + continue + } + for _, metric := range family.Metric { + for _, label := range metric.Label { + if label.GetName() == "entity_value" && label.GetValue() == entity { + return metric.GetCounter().GetValue() + } + } + } + } + + t.Fatalf("fatal event metric for %q not found", entity) + return 0 +} + func (m *mockPublisher) HealthEventOccurredV1(ctx context.Context, events *protos.HealthEvents, opts ...grpc.CallOption) (*emptypb.Empty, error) { args := m.Called(ctx, events) return args.Get(0).(*emptypb.Empty), args.Error(1) @@ -124,8 +248,9 @@ func (m *mockDatabaseClient) DeleteResumeToken(ctx context.Context, tokenConfig // Mock cursor for tests type mockCursor struct { mock.Mock - data []map[string]any - pos int + data []map[string]any + pos int + allErr error } func createMockCursor(data []map[string]any) (*mockCursor, error) { @@ -151,6 +276,10 @@ func (m *mockCursor) Close(ctx context.Context) error { } func (m *mockCursor) All(ctx context.Context, results any) error { + if m.allErr != nil { + return m.allErr + } + if resultsSlice, ok := results.(*[]map[string]any); ok { *resultsSlice = m.data } @@ -423,6 +552,67 @@ func TestHandleEvent(t *testing.T) { mockClient.AssertNotCalled(t, "Aggregate") mockPublisher.AssertNotCalled(t, "HealthEventOccurredV1") }) + + t.Run("deterministic rule failures are skipped", func(t *testing.T) { + for name, test := range map[string]struct { + rule config.HealthEventsAnalyzerRule + aggregateErr error + }{ + "invalid stage": { + rule: config.HealthEventsAnalyzerRule{ + Name: "invalid-stage", EvaluateRule: true, Stage: []string{"invalid json"}, + }, + }, + "invalid datastore query": { + rule: config.HealthEventsAnalyzerRule{ + Name: "invalid-query", EvaluateRule: true, Stage: []string{`{"$count": "count"}`}, + }, + aggregateErr: datastore.NewValidationError(datastore.ProviderPostgreSQL, "bad pipeline", nil), + }, + } { + t.Run(name, func(t *testing.T) { + mockClient := new(mockDatabaseClient) + if test.aggregateErr != nil { + mockClient.On("Aggregate", mock.Anything, mock.Anything). + Return((*mockCursor)(nil), test.aggregateErr).Once() + } + + reconciler := &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{test.rule}}, + }, + databaseClient: mockClient, + } + + published, err := reconciler.handleEvent(ctx, &healthEvent_13) + assert.NoError(t, err) + assert.False(t, published) + mockClient.AssertExpectations(t) + }) + } + }) + + t.Run("deterministic cursor decode failures are skipped", func(t *testing.T) { + mockClient := new(mockDatabaseClient) + rule := config.HealthEventsAnalyzerRule{ + Name: "decode-failure", EvaluateRule: true, Stage: []string{`{"$count":"count"}`}, + } + mockClient.On("Aggregate", mock.Anything, mock.Anything).Return(&mockCursor{ + allErr: datastore.NewSerializationError(datastore.ProviderPostgreSQL, "bad stored row", nil), + }, nil).Once() + reconciler := &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{rule}}, + }, + databaseClient: mockClient, + } + + published, err := reconciler.handleEvent(ctx, &healthEvent_13) + assert.NoError(t, err) + assert.False(t, published) + mockClient.AssertExpectations(t) + }) + t.Run("rule with EvaluateRule false is skipped", func(t *testing.T) { mockClient := new(mockDatabaseClient) mockPublisher := &mockPublisher{} @@ -472,7 +662,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.NoError(t, err) // Agent filter (1) + configured stages (2) = 3 total assert.Len(t, pipeline, 3) @@ -513,7 +703,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.NoError(t, err) // Agent filter (1) + configured stages (4) = 5 total assert.Len(t, pipeline, 5) @@ -540,7 +730,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.NoError(t, err) // Agent filter (1) + configured stages (1) = 2 total assert.Len(t, pipeline, 2) @@ -563,7 +753,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.Error(t, err) assert.Nil(t, pipeline) assert.Contains(t, err.Error(), "failed to parse stage 0") @@ -582,7 +772,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.Error(t, err) assert.Nil(t, pipeline) }) @@ -598,7 +788,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.NoError(t, err) // Even with empty stages, agent filter is always present assert.Len(t, pipeline, 1) @@ -618,7 +808,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.NoError(t, err) // Agent filter (1) + configured stages (2) = 3 total assert.Len(t, pipeline, 3) @@ -642,7 +832,7 @@ func TestGetPipelineStages(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.NoError(t, err) // Agent filter (1) + configured stages (1) = 2 total assert.Len(t, pipeline, 2) @@ -660,6 +850,23 @@ func TestGetPipelineStages(t *testing.T) { _ = ctx // suppress unused warning if any } +func TestNonRecoveryRuleExtendsOnlyMandatoryStage(t *testing.T) { + database := new(mockDatabaseClient) + database.On("Aggregate", mock.Anything, mock.MatchedBy(func(pipeline any) bool { + options, ok := pipeline.(client.PipelineOptions) + return ok && !options.EnableExtendedFilters && options.ExtendedFilterPrefix == 1 + })).Return(&mockCursor{data: nil, pos: -1}, nil).Once() + reconciler := &Reconciler{databaseClient: database} + rule := config.HealthEventsAnalyzerRule{ + Name: "legacy-rule", EvaluateRule: true, Stage: []string{`{"$count":"count"}`}, + } + + matched, err := reconciler.validateAllSequenceCriteria(context.Background(), rule, healthEvent_13) + assert.NoError(t, err) + assert.False(t, matched) + database.AssertExpectations(t) +} + // TestGetPipelineStages_ReturnTypeCompatibility ensures the pipeline return type // is []map[string]interface{} (not []interface{}) to maintain compatibility with // MongoDB's Aggregate function. This test prevents regression of the bug where @@ -688,7 +895,7 @@ func TestGetPipelineStages_ReturnTypeCompatibility(t *testing.T) { }, } - pipeline, err := reconciler.getPipelineStages(rule, event) + pipeline, err := reconciler.getPipelineStages(rule, event, nil) assert.NoError(t, err) // CRITICAL: Verify the return type is []map[string]interface{} diff --git a/health-events-analyzer/pkg/reconciler/recovery.go b/health-events-analyzer/pkg/reconciler/recovery.go new file mode 100644 index 000000000..d43669f88 --- /dev/null +++ b/health-events-analyzer/pkg/reconciler/recovery.go @@ -0,0 +1,1332 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reconciler + +import ( + "context" + "database/sql/driver" + "errors" + "fmt" + "io" + "log/slog" + "net" + "slices" + "strings" + "sync" + "time" + + multierror "github.com/hashicorp/go-multierror" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + + datamodels "github.com/nvidia/nvsentinel/data-models/pkg/model" + protos "github.com/nvidia/nvsentinel/data-models/pkg/protos" + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/config" + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/publisher" + "github.com/nvidia/nvsentinel/store-client/pkg/client" + "github.com/nvidia/nvsentinel/store-client/pkg/datastore" +) + +type recoveryIdentity struct { + key string + nodeName string + entities []*protos.Entity +} + +type recoveryBoundary struct { + createdAt time.Time + generated *timestamppb.Timestamp +} + +type derivedState struct { + boundary recoveryBoundary + isHealthy bool +} + +type recoveryTarget struct { + identity recoveryIdentity + state derivedState +} + +type storedDocumentTargetScope uint8 + +const ( + storedDocumentAffectsAllTargets storedDocumentTargetScope = iota + storedDocumentAffectsNoTargets + storedDocumentAffectsIdentity +) + +const maxStoredDocumentErrorDetails = 3 + +type storedDocumentDecodeError struct { + cause error + classification string + identityKey string + targetScope storedDocumentTargetScope +} + +func (e *storedDocumentDecodeError) Error() string { + return e.cause.Error() +} + +func (e *storedDocumentDecodeError) Unwrap() error { + return e.cause +} + +func (e *storedDocumentDecodeError) affects(identity recoveryIdentity) bool { + switch e.targetScope { + case storedDocumentAffectsNoTargets: + return false + case storedDocumentAffectsAllTargets: + return true + case storedDocumentAffectsIdentity: + return e.identityKey == identity.key + } + + return false +} + +type storedDocumentScanError struct { + issues []*storedDocumentDecodeError +} + +func (e *storedDocumentScanError) Error() string { + detailCount := min(len(e.issues), maxStoredDocumentErrorDetails) + messages := make([]string, 0, detailCount+1) + + for _, issue := range e.issues[:detailCount] { + messages = append(messages, issue.Error()) + } + + if remaining := len(e.issues) - detailCount; remaining > 0 { + messages = append(messages, fmt.Sprintf("and %d more", remaining)) + } + + return fmt.Sprintf("%d stored health event document(s) were incomplete: %s", + len(e.issues), strings.Join(messages, "; ")) +} + +func (e *storedDocumentScanError) Unwrap() []error { + errs := make([]error, 0, len(e.issues)) + for _, issue := range e.issues { + errs = append(errs, issue) + } + + return errs +} + +func (e *storedDocumentScanError) append(issue *storedDocumentDecodeError) { + e.issues = append(e.issues, issue) +} + +func (e *storedDocumentScanError) errorOrNil() error { + if len(e.issues) == 0 { + return nil + } + + return e +} + +func (e *storedDocumentScanError) affects(identity recoveryIdentity) bool { + for _, issue := range e.issues { + if issue.affects(identity) { + return true + } + } + + return false +} + +func (e *storedDocumentScanError) hasUnreadableIdentity() bool { + for _, issue := range e.issues { + if issue.targetScope == storedDocumentAffectsAllTargets { + return true + } + } + + return false +} + +func (e *storedDocumentScanError) hasInvalidIdentity() bool { + for _, issue := range e.issues { + if issue.targetScope == storedDocumentAffectsNoTargets { + return true + } + } + + return false +} + +func (e *storedDocumentScanError) skippedTargetCount( + targets []recoveryTarget, + requested recoveryIdentity, + nodeWide bool, +) int { + skipped := make(map[string]struct{}) + + for _, issue := range e.issues { + if issue.targetScope == storedDocumentAffectsIdentity { + skipped[issue.identityKey] = struct{}{} + } + } + + for _, target := range targets { + if e.affects(target.identity) { + skipped[target.identity.key] = struct{}{} + } + } + + if !nodeWide && e.affects(requested) { + skipped[requested.key] = struct{}{} + } + + return len(skipped) +} + +type storedDocumentIssueTracker struct { + mu sync.Mutex + seen map[string]struct{} +} + +type storedDocumentIssueTrackerContextKey struct{} + +func withStoredDocumentIssueTracker(ctx context.Context) context.Context { + return context.WithValue(ctx, storedDocumentIssueTrackerContextKey{}, &storedDocumentIssueTracker{ + seen: make(map[string]struct{}), + }) +} + +func (t *storedDocumentIssueTracker) mark(key string) bool { + t.mu.Lock() + defer t.mu.Unlock() + + if _, found := t.seen[key]; found { + return false + } + + t.seen[key] = struct{}{} + + return true +} + +type storedRecoveryIdentityDocument struct { + HealthEvent *storedRecoveryIdentityEvent `bson:"healthevent" json:"healthevent"` +} + +type storedRecoveryIdentityEvent struct { + NodeName string `bson:"nodename" json:"nodeName"` + EntitiesImpacted []storedRecoveryIdentityEntity `bson:"entitiesimpacted" json:"entitiesImpacted"` +} + +type storedRecoveryIdentityEntity struct { + EntityType string `bson:"entitytype" json:"entityType"` + EntityValue string `bson:"entityvalue" json:"entityValue"` +} + +const ( + defaultRecoveryPollInterval = 250 * time.Millisecond + defaultRecoveryRepublishInterval = 30 * time.Second + defaultRecoveryPersistenceTimeout = 2 * time.Minute +) + +func recoveryIdentityForEvent( + rule config.HealthEventsAnalyzerRule, + event *protos.HealthEvent, +) (recoveryIdentity, bool) { + if rule.Recovery == nil || event == nil || event.NodeName == "" { + return recoveryIdentity{}, false + } + + identity := recoveryIdentity{ + nodeName: event.NodeName, + key: event.NodeName, + } + + if rule.Recovery.Scope == config.RecoveryScopeNode { + return identity, true + } + + entities, foundAllTypes := recoveryEntities(event.EntitiesImpacted, rule.Recovery.EntityTypes) + if !foundAllTypes { + return recoveryIdentity{}, false + } + + identity.entities = entities + identity.key = recoveryEntityKey(event.NodeName, entities) + + return identity, true +} + +func recoveryIdentityForSource( + rule config.HealthEventsAnalyzerRule, + event *protos.HealthEvent, +) (identity recoveryIdentity, nodeWide bool, ok bool) { + if rule.Recovery == nil || event == nil || event.NodeName == "" { + return recoveryIdentity{}, false, false + } + + if rule.Recovery.Scope == config.RecoveryScopeNode { + identity, ok := recoveryIdentityForEvent(rule, event) + return identity, false, ok + } + + identity, ok = recoveryIdentityForEvent(rule, event) + if ok { + return identity, false, true + } + + if len(event.EntitiesImpacted) != 0 { + return recoveryIdentity{}, false, false + } + + return nodeRecoveryIdentity(event.NodeName), true, true +} + +func nodeRecoveryIdentity(nodeName string) recoveryIdentity { + return recoveryIdentity{ + nodeName: nodeName, + key: nodeName + "|*", + } +} + +func recoveryEntities(entities []*protos.Entity, entityTypes []string) ([]*protos.Entity, bool) { + allowedTypes := make(map[string]struct{}, len(entityTypes)) + for _, entityType := range entityTypes { + allowedTypes[entityType] = struct{}{} + } + + selectedByType := make(map[string]*protos.Entity, len(entityTypes)) + + for _, entity := range entities { + if !selectRecoveryEntity(selectedByType, allowedTypes, entity) { + return nil, false + } + } + + if len(selectedByType) != len(allowedTypes) { + return nil, false + } + + selected := make([]*protos.Entity, 0, len(selectedByType)) + for _, entity := range selectedByType { + selected = append(selected, entity) + } + + slices.SortFunc(selected, func(a, b *protos.Entity) int { + if result := strings.Compare(a.EntityType, b.EntityType); result != 0 { + return result + } + + return strings.Compare(a.EntityValue, b.EntityValue) + }) + + return selected, true +} + +func selectRecoveryEntity( + selected map[string]*protos.Entity, + allowed map[string]struct{}, + entity *protos.Entity, +) bool { + if entity == nil || entity.EntityValue == "" { + return true + } + + if _, ok := allowed[entity.EntityType]; !ok { + return true + } + + existing, found := selected[entity.EntityType] + if found { + return existing.EntityValue == entity.EntityValue + } + + selected[entity.EntityType] = proto.Clone(entity).(*protos.Entity) + + return true +} + +func recoveryEntityKey(nodeName string, entities []*protos.Entity) string { + var key strings.Builder + key.WriteString(nodeName) + + for _, entity := range entities { + fmt.Fprintf(&key, "|%d:%s=%d:%s", len(entity.EntityType), entity.EntityType, + len(entity.EntityValue), entity.EntityValue) + } + + return key.String() +} + +func recoverySourceMatches(mapping *config.RecoveryMapping, event *protos.HealthEvent) bool { + if mapping == nil || event == nil || !event.IsHealthy || event.CheckName != mapping.SourceCheckName { + return false + } + + if mapping.SourceAgent != "" && event.Agent != mapping.SourceAgent { + return false + } + + if len(mapping.SourceErrorCodes) == 0 { + return true + } + + for _, eventCode := range event.ErrorCode { + if slices.Contains(mapping.SourceErrorCodes, eventCode) { + return true + } + } + + return false +} + +func recoveryStateKey(ruleName string, identity recoveryIdentity) string { + return ruleName + "\x00" + identity.key +} + +func (r *Reconciler) handleRecoveryEvents( + ctx context.Context, + event *datamodels.HealthEventWithStatus, +) (bool, error) { + if event == nil || event.HealthEvent == nil || !event.HealthEvent.IsHealthy { + return false, nil + } + + published := false + + var multiErr *multierror.Error + + for _, rule := range r.config.HealthEventsAnalyzerRules.Rules { + recovered, err := r.handleRecoveryRule(ctx, event, rule) + if err != nil { + published = recovered || published + + if client.IsPermanentError(err) { + slog.ErrorContext(ctx, "Skipping recovery rule after deterministic failure", + "rule_name", rule.Name, "error", err) + totalEventProcessingError.WithLabelValues("permanent_recovery_rule_error").Inc() + + continue + } + + multiErr = multierror.Append(multiErr, err) + + continue + } + + published = recovered || published + } + + return published, multiErr.ErrorOrNil() +} + +//nolint:cyclop // Recovery keeps partial scan, publication, and boundary failures distinct. +func (r *Reconciler) handleRecoveryRule( + ctx context.Context, + event *datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, +) (bool, error) { + if !rule.EvaluateRule || !recoverySourceMatches(rule.Recovery, event.HealthEvent) { + return false, nil + } + + identity, nodeWide, ok := recoveryIdentityForSource(rule, event.HealthEvent) + if !ok { + slog.WarnContext(ctx, "Recovery event does not contain the configured scope", + "rule_name", rule.Name, + "node", event.HealthEvent.NodeName, + "entity_types", rule.Recovery.EntityTypes) + + return false, nil + } + + sourceBoundary := boundaryFromEvent(event) + + targets, targetErr := r.recoveryTargets(ctx, rule, identity, nodeWide) + scanIncomplete := false + + var scanErr *storedDocumentScanError + if errors.As(targetErr, &scanErr) { + scanIncomplete = true + skippedTargets := scanErr.skippedTargetCount(targets, identity, nodeWide) + targets = slices.DeleteFunc(targets, func(target recoveryTarget) bool { + return scanErr.affects(target.identity) + }) + checkpointSource := client.IsPermanentError(scanErr) + slog.ErrorContext(ctx, "Incomplete stored-state scan limited recovery targets", + "rule_name", rule.Name, + "skipped_targets", skippedTargets, + "unreadable_identity", scanErr.hasUnreadableIdentity(), + "invalid_identity", scanErr.hasInvalidIdentity(), + "checkpoint_source", checkpointSource, + "error", scanErr) + totalEventProcessingError.WithLabelValues("recovery_stored_document_incomplete").Inc() + + if checkpointSource { + targetErr = nil + } + } + + if targetErr != nil && len(targets) == 0 { + return false, fmt.Errorf("find current states for rule %q: %w", rule.Name, targetErr) + } + + if !nodeWide && len(targets) == 0 && !scanIncomplete { + // A verified recovery also resets rule history when there is no derived + // condition to clear yet. + r.rememberRecoveryBoundary(rule.Name, identity, sourceBoundary) + } + + published, err := r.publishRecoveryTargets(ctx, event, rule, targets, sourceBoundary, false) + if err != nil { + if targetErr != nil { + return published, errors.Join( + fmt.Errorf("find current states for rule %q: %w", rule.Name, targetErr), err, + ) + } + + return published, err + } + + if targetErr != nil { + return published, fmt.Errorf("find current states for rule %q: %w", rule.Name, targetErr) + } + + // A node-wide boundary affects every entity on the node, so expose it only + // after all applicable derived conditions have been durably recovered. + if nodeWide && !scanIncomplete { + r.rememberRecoveryBoundary(rule.Name, identity, sourceBoundary) + } + + return published, nil +} + +func (r *Reconciler) publishRecoveryTargets( + ctx context.Context, + event *datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, + targets []recoveryTarget, + sourceBoundary recoveryBoundary, + published bool, +) (bool, error) { + for _, target := range targets { + // A delayed healthy event must not clear a newer derived condition or + // move its history boundary forward. + if !boundaryAfter(sourceBoundary, target.state.boundary) { + continue + } + + if target.state.isHealthy { + r.rememberRecoveryBoundary(rule.Name, target.identity, sourceBoundary) + continue + } + + persistedBoundary, didPublish, err := r.publishRecoveryUntilStored(ctx, event, rule, target.identity) + if err != nil { + return published, fmt.Errorf("publish recovery for rule %q: %w", rule.Name, err) + } + + r.rememberRecoveryBoundary(rule.Name, target.identity, sourceBoundary) + r.rememberDerivedState(rule.Name, target.identity, derivedState{ + boundary: persistedBoundary, + isHealthy: true, + }) + + if didPublish { + recoveryEventsPublishedTotal.WithLabelValues(rule.Name, string(rule.Recovery.Scope)).Inc() + + published = true + } + } + + return published, nil +} + +func (r *Reconciler) recoveryTargets( + ctx context.Context, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, + nodeWide bool, +) ([]recoveryTarget, error) { + if nodeWide { + return r.currentDerivedStatesForNode(ctx, rule, identity.nodeName) + } + + state, found, err := r.currentDerivedState(ctx, rule, identity) + if err != nil || !found { + return nil, err + } + + return []recoveryTarget{{identity: identity, state: state}}, nil +} + +func (r *Reconciler) recoveryBoundaryForEvent( + ctx context.Context, + rule config.HealthEventsAnalyzerRule, + event *protos.HealthEvent, +) (*recoveryBoundary, error) { + identity, ok := recoveryIdentityForEvent(rule, event) + if !ok { + return nil, nil + } + + if boundary, found := r.cachedEffectiveRecoveryBoundary(rule, identity); found { + effective, err := r.recoveryBoundaryIsEffective(ctx, rule, identity, boundary) + if err != nil { + return nil, fmt.Errorf("validate cached recovery boundary: %w", err) + } + + if !effective { + return nil, nil + } + + return &boundary, nil + } + + if r.recoveryLookupLoaded(rule.Name, identity) { + return nil, nil + } + + latest, err := r.latestRecoverySource(ctx, rule, identity) + if err != nil { + return nil, fmt.Errorf("find latest recovery source: %w", err) + } + + if latest == nil { + r.rememberRecoveryLookup(rule.Name, identity) + return nil, nil + } + + boundary := boundaryFromEvent(latest) + + effective, err := r.recoveryBoundaryIsEffective(ctx, rule, identity, boundary) + if err != nil { + return nil, fmt.Errorf("find derived state for recovery boundary: %w", err) + } + + if !effective { + return nil, nil + } + + r.rememberRecoveryBoundaryFromSource(rule, identity, latest.HealthEvent, boundary) + + return &boundary, nil +} + +func (r *Reconciler) latestRecoverySource( + ctx context.Context, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, +) (*datamodels.HealthEventWithStatus, error) { + return r.findLatestMatchingEvent(ctx, &rule, &identity, rule.Name, "recovery_source", r.recoveryLookupFilter( + rule.Recovery.SourceAgent, rule.Recovery.SourceCheckName, identity.nodeName, + ), func(candidate *datamodels.HealthEventWithStatus) bool { + return recoverySourceMatches(rule.Recovery, candidate.HealthEvent) && + recoverySourceAppliesToIdentity(rule, candidate.HealthEvent, identity) + }) +} + +func (r *Reconciler) recoveryBoundaryIsEffective( + ctx context.Context, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, + boundary recoveryBoundary, +) (bool, error) { + state, found, err := r.currentDerivedState(ctx, rule, identity) + if err != nil { + return false, err + } + + // A healthy source is not an effective history boundary while a preceding + // derived condition is still unhealthy. This can happen when recovery + // publication failed and the source event was later checkpointed as poison. + return !found || state.isHealthy || !boundaryAfter(boundary, state.boundary), nil +} + +func (r *Reconciler) rememberRecoveryBoundaryFromSource( + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, + source *protos.HealthEvent, + boundary recoveryBoundary, +) { + sourceIdentity, nodeWide, valid := recoveryIdentityForSource(rule, source) + if valid && nodeWide { + r.rememberRecoveryBoundary(rule.Name, sourceIdentity, boundary) + } else { + r.rememberRecoveryBoundary(rule.Name, identity, boundary) + } +} + +func recoverySourceAppliesToIdentity( + rule config.HealthEventsAnalyzerRule, + event *protos.HealthEvent, + identity recoveryIdentity, +) bool { + sourceIdentity, nodeWide, ok := recoveryIdentityForSource(rule, event) + return ok && (nodeWide || sourceIdentity.key == identity.key) +} + +func (r *Reconciler) currentDerivedState( + ctx context.Context, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, +) (state derivedState, found bool, err error) { + if state, found := r.cachedDerivedState(rule.Name, identity); found { + return state, true, nil + } + + latest, err := r.findLatestMatchingEvent(ctx, &rule, &identity, rule.Name, "derived_state", r.recoveryLookupFilter( + agentName, rule.Name, identity.nodeName, + ), func(candidate *datamodels.HealthEventWithStatus) bool { + candidateIdentity, valid := recoveryIdentityForEvent(rule, candidate.HealthEvent) + return valid && candidateIdentity.key == identity.key + }) + if err != nil { + return derivedState{}, false, err + } + + if latest == nil { + return derivedState{}, false, nil + } + + return derivedState{ + boundary: boundaryFromEvent(latest), + isHealthy: latest.HealthEvent.IsHealthy, + }, true, nil +} + +func (r *Reconciler) currentDerivedStatesForNode( + ctx context.Context, + rule config.HealthEventsAnalyzerRule, + nodeName string, +) ([]recoveryTarget, error) { + cursor, err := r.databaseClient.Find(ctx, r.recoveryLookupFilter(agentName, rule.Name, nodeName), nil) + if err != nil { + return nil, err + } + defer func() { + if closeErr := cursor.Close(ctx); closeErr != nil { + slog.WarnContext(ctx, "Failed to close recovery cursor", + "rule", rule.Name, + "lookup", "node_derived_states", + "error", closeErr, + ) + } + }() + + states := make(map[string]recoveryTarget) + + decodeErrs := &storedDocumentScanError{} + + for cursor.Next(ctx) { + var candidate datamodels.HealthEventWithStatus + if err := cursor.Decode(&candidate); err != nil { + decodeErr := r.newStoredDocumentDecodeError(ctx, cursor, &rule, err) + r.recordStoredDocumentDecodeError(ctx, rule.Name, "node_derived_states", decodeErr) + decodeErrs.append(decodeErr) + + continue + } + + identity, valid := recoveryIdentityForEvent(rule, candidate.HealthEvent) + if !valid { + identityErr := newStoredDocumentIdentityError() + r.recordStoredDocumentDecodeError(ctx, rule.Name, "node_derived_states", identityErr) + decodeErrs.append(identityErr) + + continue + } + + state := derivedState{ + boundary: boundaryFromEvent(&candidate), + isHealthy: candidate.HealthEvent.IsHealthy, + } + + current, found := states[identity.key] + if !found || boundaryAfter(state.boundary, current.state.boundary) { + states[identity.key] = recoveryTarget{identity: identity, state: state} + } + } + + if err := cursor.Err(); err != nil { + return nil, fmt.Errorf("iterate health events: %w", err) + } + + targets := make([]recoveryTarget, 0, len(states)) + for _, target := range states { + targets = append(targets, target) + } + + slices.SortFunc(targets, func(a, b recoveryTarget) int { + return strings.Compare(a.identity.key, b.identity.key) + }) + + return targets, decodeErrs.errorOrNil() +} + +// publishRecoveryUntilStored keeps the source event in-flight until the store +// connector has durably inserted the corresponding derived recovery. The +// platform-connector RPC acknowledges an in-memory enqueue, so returning after +// the RPC alone would allow the source resume token to advance before the +// recovery is durable. +func (r *Reconciler) publishRecoveryUntilStored( + ctx context.Context, + source *datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, +) (recoveryBoundary, bool, error) { + return r.publishDerivedUntilStored( + ctx, source, rule, identity, true, "recovery", + func(publishCtx context.Context) error { + return r.config.Publisher.PublishRecovery( + publishCtx, source.HealthEvent, rule.Name, identity.entities, &rule, + ) + }, + ) +} + +func (r *Reconciler) publishFaultUntilStored( + ctx context.Context, + source *datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, +) (recoveryBoundary, bool, error) { + event := source.HealthEvent + if rule.Recovery.Scope == config.RecoveryScopeEntity { + event = proto.Clone(source.HealthEvent).(*protos.HealthEvent) + event.EntitiesImpacted = identity.entities + } + + action := protos.RecommendedAction(r.getRecommendedActionValue(rule.RecommendedAction, rule.Name)) + + return r.publishDerivedUntilStored( + ctx, source, rule, identity, false, "fault", + func(publishCtx context.Context) error { + return r.config.Publisher.Publish(publishCtx, event, action, rule.Name, rule.Message, &rule) + }, + ) +} + +func (r *Reconciler) publishDerivedUntilStored( + ctx context.Context, + source *datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, + isHealthy bool, + stateName string, + publish func(context.Context) error, +) (recoveryBoundary, bool, error) { + pollInterval, republishInterval := r.recoveryIntervals() + + timeoutCtx, cancel := context.WithTimeout(ctx, r.recoveryPersistenceTimeout()) + defer cancel() + + waitCtx := withStoredDocumentIssueTracker(timeoutCtx) + + nextPublish := time.Time{} + published := false + + for { + persisted, err := r.findPersistedDerived(waitCtx, source, rule, identity, isHealthy) + if err == nil && persisted != nil { + return boundaryFromEvent(persisted), published, nil + } + + if err != nil { + if client.IsPermanentError(err) { + return recoveryBoundary{}, published, err + } + + slog.WarnContext(waitCtx, "Failed to confirm persisted derived event; retrying", + "state", stateName, + "rule_name", rule.Name, + "node", identity.nodeName, + "error", err) + } + + var didPublish bool + + nextPublish, didPublish = publishDerivedIfDue( + waitCtx, publish, stateName, rule.Name, identity.nodeName, + nextPublish, republishInterval, + ) + published = didPublish || published + + if err := waitForRecoveryPoll(waitCtx, pollInterval); err != nil { + if waitCtx.Err() == context.DeadlineExceeded { + recoveryPersistenceTimeoutsTotal.WithLabelValues(rule.Name, stateName).Inc() + + return recoveryBoundary{}, published, fmt.Errorf( + "timed out waiting for persisted derived %s for rule %q: %w", + stateName, rule.Name, waitCtx.Err(), + ) + } + + return recoveryBoundary{}, published, err + } + } +} + +func waitForRecoveryPoll(ctx context.Context, interval time.Duration) error { + timer := time.NewTimer(interval) + defer timer.Stop() + + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (r *Reconciler) recoveryPersistenceTimeout() time.Duration { + if r.recoveryTimeout > 0 { + return r.recoveryTimeout + } + + return defaultRecoveryPersistenceTimeout +} + +func (r *Reconciler) recoveryIntervals() (time.Duration, time.Duration) { + pollInterval := r.recoveryPoll + if pollInterval <= 0 { + pollInterval = defaultRecoveryPollInterval + } + + republishInterval := r.recoveryRepublish + if republishInterval <= 0 { + republishInterval = defaultRecoveryRepublishInterval + } + + return pollInterval, republishInterval +} + +func publishDerivedIfDue( + ctx context.Context, + publish func(context.Context) error, + stateName string, + ruleName string, + nodeName string, + nextPublish time.Time, + republishInterval time.Duration, +) (time.Time, bool) { + now := time.Now() + if nextPublish.After(now) { + return nextPublish, false + } + + err := publish(ctx) + if err != nil { + slog.WarnContext(ctx, "Failed to enqueue derived event; retrying", + "state", stateName, + "rule_name", ruleName, + "node", nodeName, + "error", err) + + return now.Add(republishInterval), false + } + + return now.Add(republishInterval), true +} + +func (r *Reconciler) findPersistedDerived( + ctx context.Context, + source *datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, + isHealthy bool, +) (*datamodels.HealthEventWithStatus, error) { + return r.findLatestMatchingEvent(ctx, &rule, &identity, rule.Name, "persisted_derived", r.recoveryLookupFilter( + agentName, rule.Name, identity.nodeName, + ), func(candidate *datamodels.HealthEventWithStatus) bool { + if candidate.HealthEvent.IsHealthy != isHealthy || + !sameRecoverySource(candidate, source) { + return false + } + + candidateIdentity, valid := recoveryIdentityForEvent(rule, candidate.HealthEvent) + + return valid && candidateIdentity.key == identity.key + }) +} + +func sameRecoverySource( + candidate *datamodels.HealthEventWithStatus, + source *datamodels.HealthEventWithStatus, +) bool { + if candidate == nil || candidate.HealthEvent == nil || source == nil || source.HealthEvent == nil { + return false + } + + if !recoverySourceTimestampMatches(candidate.HealthEvent, source.HealthEvent) { + return false + } + + return source.CreatedAt.IsZero() || !candidate.CreatedAt.Before(source.CreatedAt) +} + +func recoverySourceTimestampMatches(candidate, source *protos.HealthEvent) bool { + sourceTimestamp := source.GeneratedTimestamp + preservedTimestamp, hasPreservedTimestamp := candidate.Metadata[publisher.SourceGeneratedTimestampMetadataKey] + + if sourceTimestamp == nil { + // Current publishers cannot preserve a timestamp that the source did not + // provide, so datastore order is the only available correlation signal. + return !hasPreservedTimestamp + } + + if hasPreservedTimestamp { + return preservedTimestamp == sourceTimestamp.AsTime().UTC().Format(time.RFC3339Nano) + } + + // Backward compatibility for derived events persisted before publishers began + // stamping their own generated timestamp and preserving the source in metadata. + return proto.Equal(candidate.GeneratedTimestamp, sourceTimestamp) +} + +func boundaryAfter(candidate, current recoveryBoundary) bool { + if candidate.generated != nil && current.generated != nil { + candidateTime := candidate.generated.AsTime() + currentTime := current.generated.AsTime() + + if !candidateTime.Equal(currentTime) { + return candidateTime.After(currentTime) + } + } + + if !candidate.createdAt.IsZero() && !current.createdAt.IsZero() { + return candidate.createdAt.After(current.createdAt) + } + + return false +} + +func (r *Reconciler) recoveryLookupFilter(agent, checkName, nodeName string) map[string]any { + checkNameField := "healthevent.checkname" + nodeNameField := "healthevent.nodename" + + if r.provider == datastore.ProviderPostgreSQL { + checkNameField = "event_type" + nodeNameField = "node_name" + } + + filter := map[string]any{ + checkNameField: checkName, + nodeNameField: nodeName, + } + if agent != "" { + filter["healthevent.agent"] = agent + } + + return filter +} + +//nolint:cyclop // The scan keeps decode, identity, match, and iteration failures distinct. +func (r *Reconciler) findLatestMatchingEvent( + ctx context.Context, + rule *config.HealthEventsAnalyzerRule, + identity *recoveryIdentity, + ruleName string, + lookup string, + filter map[string]any, + matches func(*datamodels.HealthEventWithStatus) bool, +) (*datamodels.HealthEventWithStatus, error) { + cursor, err := r.databaseClient.Find(ctx, filter, &client.FindOptions{ + Sort: map[string]any{"createdAt": -1}, + }) + if err != nil { + return nil, err + } + defer func() { + if closeErr := cursor.Close(ctx); closeErr != nil { + slog.WarnContext(ctx, "Failed to close recovery cursor", + "rule", ruleName, + "lookup", lookup, + "error", closeErr, + ) + } + }() + + var latest *datamodels.HealthEventWithStatus + + decodeErrs := &storedDocumentScanError{} + + for cursor.Next(ctx) { + var candidate datamodels.HealthEventWithStatus + if err := cursor.Decode(&candidate); err != nil { + decodeErr := r.newStoredDocumentDecodeError(ctx, cursor, rule, err) + r.recordStoredDocumentDecodeError(ctx, ruleName, lookup, decodeErr) + + if identity == nil || decodeErr.affects(*identity) { + decodeErrs.append(decodeErr) + } + + continue + } + + if candidate.HealthEvent == nil { + // A decoded row without a health event cannot match this identity. + // Count it, but do not let it abort an otherwise valid identity lookup. + r.recordStoredDocumentDecodeError(ctx, ruleName, lookup, newStoredDocumentIdentityError()) + + continue + } + + if !matches(&candidate) { + continue + } + + if latest == nil || boundaryAfter(boundaryFromEvent(&candidate), boundaryFromEvent(latest)) { + matched := candidate + latest = &matched + } + } + + if err := cursor.Err(); err != nil { + return nil, fmt.Errorf("iterate health events: %w", err) + } + + return latest, decodeErrs.errorOrNil() +} + +func (r *Reconciler) recordStoredDocumentDecodeError( + ctx context.Context, + ruleName string, + lookup string, + issue *storedDocumentDecodeError, +) { + key := strings.Join([]string{ + ruleName, lookup, issue.classification, issue.identityKey, issue.Error(), + }, "\x00") + if tracker, ok := ctx.Value(storedDocumentIssueTrackerContextKey{}).(*storedDocumentIssueTracker); ok && + !tracker.mark(key) { + return + } + + slog.ErrorContext(ctx, "Skipping unusable stored health event", + "rule_name", ruleName, "lookup", lookup, "classification", issue.classification, "error", issue) + recoveryStoredDocumentDecodeErrorsTotal.WithLabelValues(ruleName, lookup, issue.classification).Inc() +} + +func (r *Reconciler) newStoredDocumentDecodeError( + ctx context.Context, + cursor client.Cursor, + rule *config.HealthEventsAnalyzerRule, + err error, +) *storedDocumentDecodeError { + cause := classifyRecoveryDecodeError(ctx, err) + + targetScope := storedDocumentAffectsAllTargets + identity := recoveryIdentity{} + + if rule != nil { + var identityDecoded, identityValid bool + + identity, identityDecoded, identityValid = recoveryIdentityFromCurrentDocument(cursor, *rule) + + if identityDecoded { + targetScope = storedDocumentAffectsNoTargets + if identityValid { + targetScope = storedDocumentAffectsIdentity + } + } + } + + classification := "transient" + if client.IsPermanentError(cause) { + classification = "malformed" + } + + return &storedDocumentDecodeError{ + cause: cause, + classification: classification, + identityKey: identity.key, + targetScope: targetScope, + } +} + +func newStoredDocumentIdentityError() *storedDocumentDecodeError { + return &storedDocumentDecodeError{ + cause: client.PermanentError(errors.New("stored health event has no valid recovery identity")), + classification: "invalid_identity", + targetScope: storedDocumentAffectsNoTargets, + } +} + +func recoveryIdentityFromCurrentDocument( + cursor client.Cursor, + rule config.HealthEventsAnalyzerRule, +) (recoveryIdentity, bool, bool) { + var document storedRecoveryIdentityDocument + if err := cursor.Decode(&document); err != nil { + // Without readable identity fields, no target can be proven safe. Fail + // closed for every target on the node; error classification separately + // determines whether the source is checkpointed or replayed. + return recoveryIdentity{}, false, false + } + + if document.HealthEvent == nil { + return recoveryIdentity{}, true, false + } + + event := &protos.HealthEvent{NodeName: document.HealthEvent.NodeName} + + event.EntitiesImpacted = make([]*protos.Entity, 0, len(document.HealthEvent.EntitiesImpacted)) + + for _, entity := range document.HealthEvent.EntitiesImpacted { + event.EntitiesImpacted = append(event.EntitiesImpacted, &protos.Entity{ + EntityType: entity.EntityType, EntityValue: entity.EntityValue, + }) + } + + identity, valid := recoveryIdentityForEvent(rule, event) + + return identity, true, valid +} + +func classifyRecoveryDecodeError(ctx context.Context, err error) error { + wrapped := fmt.Errorf("decode health event: %w", err) + + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("%w: %w", wrapped, ctxErr) + } + + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, driver.ErrBadConn) || errors.Is(err, io.ErrUnexpectedEOF) { + return wrapped + } + + var networkError net.Error + if errors.As(err, &networkError) { + return wrapped + } + + return client.PermanentError(wrapped) +} + +func boundaryFromEvent(event *datamodels.HealthEventWithStatus) recoveryBoundary { + boundary := recoveryBoundary{createdAt: event.CreatedAt} + + if event.HealthEvent != nil && event.HealthEvent.GeneratedTimestamp != nil && + event.HealthEvent.GeneratedTimestamp.CheckValid() == nil { + boundary.generated = proto.Clone(event.HealthEvent.GeneratedTimestamp).(*timestamppb.Timestamp) + } + + return boundary +} + +func (r *Reconciler) rememberRecoveryBoundary( + ruleName string, + identity recoveryIdentity, + boundary recoveryBoundary, +) { + r.recoveryMu.Lock() + defer r.recoveryMu.Unlock() + + if r.recoveryBoundaries == nil { + r.recoveryBoundaries = make(map[string]recoveryBoundary) + } + + if r.recoveryLoaded == nil { + r.recoveryLoaded = make(map[string]struct{}) + } + + key := recoveryStateKey(ruleName, identity) + r.recoveryLoaded[key] = struct{}{} + + current, exists := r.recoveryBoundaries[key] + if !exists || boundaryAfter(boundary, current) { + r.recoveryBoundaries[key] = boundary + } +} + +func (r *Reconciler) rememberRecoveryLookup(ruleName string, identity recoveryIdentity) { + r.recoveryMu.Lock() + defer r.recoveryMu.Unlock() + + if r.recoveryLoaded == nil { + r.recoveryLoaded = make(map[string]struct{}) + } + + r.recoveryLoaded[recoveryStateKey(ruleName, identity)] = struct{}{} +} + +func (r *Reconciler) recoveryLookupLoaded(ruleName string, identity recoveryIdentity) bool { + r.recoveryMu.RLock() + defer r.recoveryMu.RUnlock() + + _, found := r.recoveryLoaded[recoveryStateKey(ruleName, identity)] + + return found +} + +func (r *Reconciler) cachedRecoveryBoundary( + ruleName string, + identity recoveryIdentity, +) (recoveryBoundary, bool) { + r.recoveryMu.RLock() + defer r.recoveryMu.RUnlock() + + boundary, found := r.recoveryBoundaries[recoveryStateKey(ruleName, identity)] + + return boundary, found +} + +func (r *Reconciler) cachedEffectiveRecoveryBoundary( + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, +) (recoveryBoundary, bool) { + // Cached boundaries are candidates, not pre-approved answers. The caller + // must run recoveryBoundaryIsEffective for the exact rule and identity + // before serving any candidate, including a node-wide one. + boundary, found := r.cachedRecoveryBoundary(rule.Name, identity) + if rule.Recovery == nil || rule.Recovery.Scope != config.RecoveryScopeEntity { + return boundary, found + } + + nodeBoundary, nodeFound := r.cachedRecoveryBoundary(rule.Name, nodeRecoveryIdentity(identity.nodeName)) + if !nodeFound || (found && !boundaryAfter(nodeBoundary, boundary)) { + return boundary, found + } + + return nodeBoundary, true +} + +func (r *Reconciler) rememberDerivedState( + ruleName string, + identity recoveryIdentity, + state derivedState, +) { + r.recoveryMu.Lock() + defer r.recoveryMu.Unlock() + + if r.derivedStates == nil { + r.derivedStates = make(map[string]derivedState) + } + + r.derivedStates[recoveryStateKey(ruleName, identity)] = state +} + +func (r *Reconciler) cachedDerivedState( + ruleName string, + identity recoveryIdentity, +) (derivedState, bool) { + r.recoveryMu.RLock() + defer r.recoveryMu.RUnlock() + + state, found := r.derivedStates[recoveryStateKey(ruleName, identity)] + + return state, found +} diff --git a/health-events-analyzer/pkg/reconciler/recovery_integration_test.go b/health-events-analyzer/pkg/reconciler/recovery_integration_test.go new file mode 100644 index 000000000..7a6aedd99 --- /dev/null +++ b/health-events-analyzer/pkg/reconciler/recovery_integration_test.go @@ -0,0 +1,502 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reconciler + +import ( + "context" + "errors" + "fmt" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + + datamodels "github.com/nvidia/nvsentinel/data-models/pkg/model" + protos "github.com/nvidia/nvsentinel/data-models/pkg/protos" + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/config" + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/publisher" + "github.com/nvidia/nvsentinel/store-client/pkg/client" + "github.com/nvidia/nvsentinel/store-client/pkg/datastore" + _ "github.com/nvidia/nvsentinel/store-client/pkg/datastore/providers" +) + +const recoveryIntegrationEnv = "NVSENTINEL_RUN_RECOVERY_INTEGRATION" + +type integrationStoreSink struct { + database client.DatabaseClient + calls atomic.Int32 + results chan error + drop map[int32]bool +} + +type recoveryObservingWatcher struct { + client.ChangeStreamWatcher + reconciler *Reconciler + source *datamodels.HealthEventWithStatus + rule config.HealthEventsAnalyzerRule + identity recoveryIdentity + started chan struct{} + marked chan error +} + +func (w *recoveryObservingWatcher) Start(ctx context.Context) { + w.ChangeStreamWatcher.Start(ctx) + close(w.started) +} + +func (w *recoveryObservingWatcher) MarkProcessed(ctx context.Context, token []byte) error { + persisted, err := w.reconciler.findPersistedDerived(ctx, w.source, w.rule, w.identity, true) + if err == nil && persisted == nil { + err = errors.New("resume token reached MarkProcessed before recovery was stored") + } + if err == nil { + err = w.ChangeStreamWatcher.MarkProcessed(ctx, token) + } + + w.marked <- err + + return err +} + +func (s *integrationStoreSink) HealthEventOccurredV1( + _ context.Context, + events *protos.HealthEvents, + _ ...grpc.CallOption, +) (*emptypb.Empty, error) { + call := s.calls.Add(1) + drop := call == 1 + if s.drop != nil { + drop = s.drop[call] + } + + if drop { + // Model an accepted ring-buffer item that is lost before the store sink + // inserts it. The reconciler must detect and republish it. + return &emptypb.Empty{}, nil + } + + documents := make([]any, 0, len(events.Events)) + for _, event := range events.Events { + documents = append(documents, datamodels.HealthEventWithStatus{ + CreatedAt: time.Now().UTC(), + HealthEvent: proto.Clone(event).(*protos.HealthEvent), + HealthEventStatus: &protos.HealthEventStatus{}, + }) + } + + go func() { + time.Sleep(5 * time.Millisecond) + _, err := s.database.InsertMany(context.Background(), documents) + s.results <- err + }() + + return &emptypb.Empty{}, nil +} + +func TestRecoveryLifecycleWithRealProvider(t *testing.T) { + if os.Getenv(recoveryIntegrationEnv) != "1" { + t.Skipf("set %s=1 with a real provider configuration", recoveryIntegrationEnv) + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + dsConfig, err := datastore.LoadDatastoreConfig() + require.NoError(t, err) + ds, err := datastore.NewDataStore(ctx, *dsConfig) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, ds.Close(context.Background())) }) + + adapter, ok := ds.(interface { + GetDatabaseClient() client.DatabaseClient + }) + require.True(t, ok) + database := adapter.GetDatabaseClient() + require.NoError(t, database.Ping(ctx)) + + runID := time.Now().UTC().UnixNano() + nodeName := fmt.Sprintf("recovery-e2e-node-%d", runID) + gpuUUID := fmt.Sprintf("GPU-%d", runID) + baseTime := time.Now().UTC().Add(-time.Minute) + rule := recoveryRule(config.RecoveryScopeEntity) + // A successful GPU-reset event identifies the recovered GPU by entity and + // does not carry an error code. + rule.Recovery.SourceErrorCodes = nil + rule.Stage = []string{ + `{"$match":{"healthevent.checkname":"SysLogsXIDError","healthevent.ishealthy":false}}`, + `{"$count":"count"}`, + `{"$match":{"count":{"$gte":2}}}`, + } + + oldEvent := func(offset time.Duration) datamodels.HealthEventWithStatus { + createdAt := time.Now().UTC() + return storedEvent(createdAt, &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + IsHealthy: false, + ErrorCode: []string{"94"}, + NodeName: nodeName, + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: gpuUUID}}, + GeneratedTimestamp: timestamppb.New(baseTime.Add(offset)), + ProcessingStrategy: protos.ProcessingStrategy_EXECUTE_REMEDIATION, + }) + } + + oldOne := oldEvent(0) + insertHealthEvents(t, ctx, database, oldOne) + oldTwo := oldEvent(10 * time.Second) + insertHealthEvents(t, ctx, database, oldTwo) + + active := derivedEvent(time.Now().UTC(), false, gpuUUID) + active.HealthEvent.NodeName = nodeName + active.HealthEvent.GeneratedTimestamp = timestamppb.New(baseTime.Add(20 * time.Second)) + insertHealthEvents(t, ctx, database, active) + + recovery := recoverySource(time.Now().UTC(), gpuUUID) + recovery.HealthEvent.ErrorCode = nil + recovery.HealthEvent.NodeName = nodeName + recovery.HealthEvent.GeneratedTimestamp = timestamppb.New(baseTime.Add(30 * time.Second)) + insertHealthEvents(t, ctx, database, recovery) + + sink := &integrationStoreSink{ + database: database, + results: make(chan error, 4), + drop: map[int32]bool{1: true, 3: true}, + } + reconciler := &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{rule}}, + Publisher: publisher.NewPublisher( + sink, + protos.ProcessingStrategy_EXECUTE_REMEDIATION, + ), + }, + databaseClient: database, + provider: ds.Provider(), + recoveryPoll: 2 * time.Millisecond, + recoveryRepublish: 100 * time.Millisecond, + } + + published, err := reconciler.handleEvent(ctx, &recovery) + require.NoError(t, err) + require.True(t, published) + require.EqualValues(t, 2, sink.calls.Load(), "lost accepted recovery must be republished") + require.NoError(t, <-sink.results) + requireStoredDerivedState(t, ctx, reconciler, rule, nodeName, gpuUUID, true) + + postRecoveryOne := oldEvent(40 * time.Second) + insertHealthEvents(t, ctx, database, postRecoveryOne) + published, err = reconciler.handleEvent(ctx, &postRecoveryOne) + require.NoError(t, err) + require.False(t, published, "pre-recovery history must not satisfy the threshold") + + postRecoveryTwo := oldEvent(50 * time.Second) + insertHealthEvents(t, ctx, database, postRecoveryTwo) + published, err = reconciler.handleEvent(ctx, &postRecoveryTwo) + require.NoError(t, err) + require.True(t, published, "two post-recovery events must reactivate the rule") + require.NoError(t, <-sink.results) + requireStoredDerivedState(t, ctx, reconciler, rule, nodeName, gpuUUID, false) + + published, err = reconciler.handleEvent(ctx, &postRecoveryTwo) + require.NoError(t, err) + require.False(t, published, "replayed source must reuse the persisted derived event") + require.EqualValues(t, 4, sink.calls.Load(), "replay must not enqueue a duplicate") +} + +func TestNonRecoveryRuleWithRealProvider(t *testing.T) { + if os.Getenv(recoveryIntegrationEnv) != "1" { + t.Skipf("set %s=1 with a real provider configuration", recoveryIntegrationEnv) + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + dsConfig, err := datastore.LoadDatastoreConfig() + require.NoError(t, err) + ds, err := datastore.NewDataStore(ctx, *dsConfig) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, ds.Close(context.Background())) }) + + adapter, ok := ds.(interface { + GetDatabaseClient() client.DatabaseClient + }) + require.True(t, ok) + database := adapter.GetDatabaseClient() + require.NoError(t, database.Ping(ctx)) + + runID := time.Now().UTC().UnixNano() + event := storedEvent(time.Now().UTC(), &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + IsHealthy: false, + NodeName: fmt.Sprintf("non-recovery-node-%d", runID), + GeneratedTimestamp: timestamppb.Now(), + // UNSPECIFIED is omitted by PostgreSQL protobuf JSON and therefore + // exercises the mandatory field-presence branch as well as top-level $or. + ProcessingStrategy: protos.ProcessingStrategy_UNSPECIFIED, + }) + insertHealthEvents(t, ctx, database, event) + + rule := config.HealthEventsAnalyzerRule{ + Name: "NonRecoveryThreshold", + EvaluateRule: true, + RecommendedAction: "CONTACT_SUPPORT", + Stage: []string{ + `{"$match":{"healthevent.checkname":"SysLogsXIDError"}}`, + `{"$count":"count"}`, + `{"$match":{"count":{"$gte":2}}}`, + }, + } + reconciler := &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{rule}}, + }, + databaseClient: database, + provider: ds.Provider(), + } + + published, err := reconciler.handleEvent(ctx, &event) + require.NoError(t, err) + require.False(t, published) +} + +func TestRecoveryWatcherAcknowledgesAfterStorageWithRealProvider(t *testing.T) { + if os.Getenv(recoveryIntegrationEnv) != "1" { + t.Skipf("set %s=1 with a real provider configuration", recoveryIntegrationEnv) + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + dsConfig, err := datastore.LoadDatastoreConfig() + require.NoError(t, err) + ds, err := datastore.NewDataStore(ctx, *dsConfig) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, ds.Close(context.Background())) }) + + adapter, ok := ds.(interface { + GetDatabaseClient() client.DatabaseClient + }) + require.True(t, ok) + database := adapter.GetDatabaseClient() + require.NoError(t, database.Ping(ctx)) + + runID := time.Now().UTC().UnixNano() + nodeName := fmt.Sprintf("recovery-watch-node-%d", runID) + gpuUUID := fmt.Sprintf("GPU-%d", runID) + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + active := derivedEvent(time.Now().UTC(), false, gpuUUID) + active.HealthEvent.NodeName = nodeName + active.HealthEvent.GeneratedTimestamp = timestamppb.New(time.Now().UTC().Add(-time.Minute)) + insertHealthEvents(t, ctx, database, active) + + source := recoverySource(time.Now().UTC(), gpuUUID) + source.HealthEvent.ErrorCode = nil + source.HealthEvent.NodeName = nodeName + source.HealthEvent.GeneratedTimestamp = timestamppb.Now() + source.HealthEvent.ProcessingStrategy = protos.ProcessingStrategy_EXECUTE_REMEDIATION + identity, ok := recoveryIdentityForEvent(rule, source.HealthEvent) + require.True(t, ok) + + sink := &integrationStoreSink{ + database: database, + results: make(chan error, 2), + } + reconciler := &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{rule}}, + Publisher: publisher.NewPublisher( + sink, + protos.ProcessingStrategy_EXECUTE_REMEDIATION, + ), + }, + databaseClient: database, + provider: ds.Provider(), + recoveryPoll: 2 * time.Millisecond, + recoveryRepublish: 100 * time.Millisecond, + } + + pipeline := analyzerPipelineForNode(nodeName) + var providerPipeline any = pipeline + if ds.Provider() == datastore.ProviderMongoDB { + providerPipeline, err = client.ConvertAgnosticPipelineToMongo(pipeline) + require.NoError(t, err) + } + + watcher, err := database.NewChangeStreamWatcher(ctx, client.TokenConfig{ + ClientName: fmt.Sprintf("recovery-e2e-%d", runID), + TokenDatabase: dsConfig.Connection.Database, + TokenCollection: "ResumeTokens", + }, providerPipeline) + require.NoError(t, err) + observingWatcher := &recoveryObservingWatcher{ + ChangeStreamWatcher: watcher, + reconciler: reconciler, + source: &source, + rule: rule, + identity: identity, + started: make(chan struct{}), + marked: make(chan error, 1), + } + + processor := client.NewEventProcessor(observingWatcher, database, client.EventProcessorConfig{ + MarkProcessedOnError: false, + }) + processor.SetEventHandler(client.EventHandlerFunc(reconciler.processHealthEvent)) + processorDone := make(chan error, 1) + go func() { processorDone <- processor.Start(ctx) }() + <-observingWatcher.started + time.Sleep(100 * time.Millisecond) + + insertHealthEvents(t, ctx, database, source) + + select { + case err := <-observingWatcher.marked: + require.NoError(t, err) + case <-ctx.Done(): + t.Fatal("timed out waiting for source resume token") + } + require.EqualValues(t, 2, sink.calls.Load()) + require.NoError(t, <-sink.results) + requireStoredDerivedState(t, ctx, reconciler, rule, nodeName, gpuUUID, true) + + cancel() + require.ErrorIs(t, <-processorDone, context.Canceled) +} + +func TestRecoveryQueryIgnoresNonNumericRowsWithRealPostgreSQL(t *testing.T) { + if os.Getenv(recoveryIntegrationEnv) != "1" { + t.Skipf("set %s=1 with a real PostgreSQL configuration", recoveryIntegrationEnv) + } + + dsConfig, err := datastore.LoadDatastoreConfig() + require.NoError(t, err) + if dsConfig.Provider != datastore.ProviderPostgreSQL { + t.Skip("PostgreSQL-specific numeric-cast regression") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + ds, err := datastore.NewDataStore(ctx, *dsConfig) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, ds.Close(context.Background())) }) + + adapter, ok := ds.(interface { + GetDatabaseClient() client.DatabaseClient + }) + require.True(t, ok) + database := adapter.GetDatabaseClient() + require.NoError(t, database.Ping(ctx)) + + runID := time.Now().UTC().UnixNano() + nodeName := fmt.Sprintf("recovery-poison-node-%d", runID) + event := storedEvent(time.Now().UTC(), &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + IsHealthy: false, + NodeName: nodeName, + GeneratedTimestamp: timestamppb.Now(), + ProcessingStrategy: protos.ProcessingStrategy_EXECUTE_REMEDIATION, + RecommendedAction: protos.RecommendedAction_CONTACT_SUPPORT, + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: fmt.Sprintf("GPU-%d", runID)}}, + }) + insertHealthEvents(t, ctx, database, event) + + updated, err := database.UpdateDocument(ctx, + map[string]any{"healthevent.nodename": nodeName}, + map[string]any{"$set": map[string]any{"healthevent.custommetric": "not-a-number"}}, + ) + require.NoError(t, err) + require.EqualValues(t, 1, updated.ModifiedCount) + + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Stage = []string{ + `{"$match":{"healthevent.custommetric":{"$gte":1}}}`, + } + reconciler := &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{rule}}, + }, + databaseClient: database, + provider: ds.Provider(), + } + + matched, err := reconciler.handleEvent(ctx, &event) + require.NoError(t, err, "non-numeric stored values must not crash-loop numeric comparisons") + require.False(t, matched) +} + +func analyzerPipelineForNode(nodeName string) datastore.Pipeline { + pipeline := client.GetPipelineBuilder().BuildAnalyzerHealthEventInsertsPipeline() + match := pipeline[0][0].Value.(datastore.Document) + match = append(match, datastore.E("fullDocument.healthevent.nodename", nodeName)) + pipeline[0][0].Value = match + + return pipeline +} + +func insertHealthEvents( + t *testing.T, + ctx context.Context, + database client.DatabaseClient, + events ...datamodels.HealthEventWithStatus, +) { + t.Helper() + + documents := make([]any, len(events)) + for i := range events { + documents[i] = events[i] + } + + _, err := database.InsertMany(ctx, documents) + require.NoError(t, err) +} + +func requireStoredDerivedState( + t *testing.T, + ctx context.Context, + reconciler *Reconciler, + rule config.HealthEventsAnalyzerRule, + nodeName string, + gpuUUID string, + wantHealthy bool, +) { + t.Helper() + + identity, ok := recoveryIdentityForEvent(rule, &protos.HealthEvent{ + NodeName: nodeName, + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: gpuUUID}}, + }) + require.True(t, ok) + + latest, err := reconciler.findLatestMatchingEvent(ctx, &rule, &identity, rule.Name, "integration", reconciler.recoveryLookupFilter( + agentName, rule.Name, nodeName, + ), func(candidate *datamodels.HealthEventWithStatus) bool { + candidateIdentity, valid := recoveryIdentityForEvent(rule, candidate.HealthEvent) + return valid && candidateIdentity.key == identity.key + }) + require.NoError(t, err) + require.NotNil(t, latest) + require.Equal(t, wantHealthy, latest.HealthEvent.IsHealthy) +} diff --git a/health-events-analyzer/pkg/reconciler/recovery_test.go b/health-events-analyzer/pkg/reconciler/recovery_test.go new file mode 100644 index 000000000..c9ee9cc3f --- /dev/null +++ b/health-events-analyzer/pkg/reconciler/recovery_test.go @@ -0,0 +1,1973 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reconciler + +import ( + "context" + "database/sql/driver" + "encoding/json" + "errors" + "fmt" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "go.mongodb.org/mongo-driver/v2/bson" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + + datamodels "github.com/nvidia/nvsentinel/data-models/pkg/model" + protos "github.com/nvidia/nvsentinel/data-models/pkg/protos" + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/config" + "github.com/nvidia/nvsentinel/health-events-analyzer/pkg/publisher" + "github.com/nvidia/nvsentinel/store-client/pkg/client" + "github.com/nvidia/nvsentinel/store-client/pkg/datastore" +) + +type healthEventCursor struct { + events []datamodels.HealthEventWithStatus + pos int + err error + decodeErr error + decodeErrs map[int]error + identityDecodeErrs map[int]error +} + +func newHealthEventCursor(events ...datamodels.HealthEventWithStatus) *healthEventCursor { + return &healthEventCursor{events: events, pos: -1} +} + +func (c *healthEventCursor) Next(context.Context) bool { + c.pos++ + return c.pos < len(c.events) +} + +func (c *healthEventCursor) Decode(value any) error { + if c.pos < 0 || c.pos >= len(c.events) { + return nil + } + + if target, ok := value.(*storedRecoveryIdentityDocument); ok { + if err := c.identityDecodeErrs[c.pos]; err != nil { + return err + } + + event := c.events[c.pos].HealthEvent + if event == nil { + return nil + } + + target.HealthEvent = &storedRecoveryIdentityEvent{NodeName: event.NodeName} + for _, entity := range event.EntitiesImpacted { + if entity == nil { + continue + } + + target.HealthEvent.EntitiesImpacted = append(target.HealthEvent.EntitiesImpacted, + storedRecoveryIdentityEntity{ + EntityType: entity.EntityType, EntityValue: entity.EntityValue, + }) + } + + return nil + } + + if err := c.decodeErrs[c.pos]; err != nil { + return err + } + + if c.decodeErr != nil { + return c.decodeErr + } + + target := value.(*datamodels.HealthEventWithStatus) + *target = c.events[c.pos] + return nil +} + +func (c *healthEventCursor) Close(context.Context) error { return nil } +func (c *healthEventCursor) All(context.Context, any) error { return nil } +func (c *healthEventCursor) Err() error { return c.err } + +type recoveryDocumentSetDatabase struct { + *mockDatabaseClient + events []datamodels.HealthEventWithStatus + decodeErrs map[int]error + identityDecodeErrs map[int]error + findCalls int +} + +func newRecoveryDocumentSetDatabase( + events []datamodels.HealthEventWithStatus, + decodeErrs map[int]error, +) *recoveryDocumentSetDatabase { + return &recoveryDocumentSetDatabase{ + mockDatabaseClient: new(mockDatabaseClient), + events: events, + decodeErrs: decodeErrs, + } +} + +func (d *recoveryDocumentSetDatabase) Find( + context.Context, + any, + *client.FindOptions, +) (client.Cursor, error) { + d.findCalls++ + events := append([]datamodels.HealthEventWithStatus(nil), d.events...) + decodeErrs := make(map[int]error, len(d.decodeErrs)) + for index, err := range d.decodeErrs { + decodeErrs[index] = err + } + identityDecodeErrs := make(map[int]error, len(d.identityDecodeErrs)) + for index, err := range d.identityDecodeErrs { + identityDecodeErrs[index] = err + } + + return &healthEventCursor{ + events: events, pos: -1, decodeErrs: decodeErrs, identityDecodeErrs: identityDecodeErrs, + }, nil +} + +func (d *recoveryDocumentSetDatabase) append(event datamodels.HealthEventWithStatus) { + d.events = append(d.events, event) +} + +type rawRecoveryIdentityCursor struct { + decode func(any) error +} + +func (c *rawRecoveryIdentityCursor) Next(context.Context) bool { return false } +func (c *rawRecoveryIdentityCursor) Decode(value any) error { return c.decode(value) } +func (c *rawRecoveryIdentityCursor) Close(context.Context) error { return nil } +func (c *rawRecoveryIdentityCursor) All(context.Context, any) error { return nil } +func (c *rawRecoveryIdentityCursor) Err() error { return nil } + +func recoveryRule(scope config.RecoveryScope) config.HealthEventsAnalyzerRule { + rule := config.HealthEventsAnalyzerRule{ + Name: "RepeatedXID94OnSameGPU", + EvaluateRule: true, + RecommendedAction: "CONTACT_SUPPORT", + Message: "Repeated XID 94", + Stage: []string{`{"$count":"count"}`}, + Recovery: &config.RecoveryMapping{ + SourceAgent: "syslog-health-monitor", + SourceCheckName: "SysLogsXIDError", + SourceErrorCodes: []string{"94"}, + Scope: scope, + }, + } + + if scope == config.RecoveryScopeEntity { + rule.Recovery.EntityTypes = []string{"GPU_UUID"} + } + + return rule +} + +func storedEvent(createdAt time.Time, event *protos.HealthEvent) datamodels.HealthEventWithStatus { + return datamodels.HealthEventWithStatus{ + CreatedAt: createdAt, + HealthEvent: event, + HealthEventStatus: &protos.HealthEventStatus{}, + } +} + +func mustRecoveryIdentity( + t *testing.T, + rule config.HealthEventsAnalyzerRule, + event *protos.HealthEvent, +) recoveryIdentity { + t.Helper() + + identity, ok := recoveryIdentityForEvent(rule, event) + require.True(t, ok) + + return identity +} + +func recoverySource(createdAt time.Time, gpuUUID string) datamodels.HealthEventWithStatus { + return storedEvent(createdAt, &protos.HealthEvent{ + Agent: "syslog-health-monitor", + ComponentClass: "GPU", + CheckName: "SysLogsXIDError", + IsHealthy: true, + ErrorCode: []string{"94"}, + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: gpuUUID}}, + GeneratedTimestamp: timestamppb.New(createdAt.Add(-time.Second)), + }) +} + +func nodeWideRecoverySource(createdAt time.Time) datamodels.HealthEventWithStatus { + event := recoverySource(createdAt, "").HealthEvent + event.ErrorCode = nil + event.EntitiesImpacted = nil + + return storedEvent(createdAt, event) +} + +func derivedEvent(createdAt time.Time, isHealthy bool, gpuUUID string) datamodels.HealthEventWithStatus { + return storedEvent(createdAt, &protos.HealthEvent{ + Agent: agentName, + ComponentClass: "GPU", + CheckName: "RepeatedXID94OnSameGPU", + IsHealthy: isHealthy, + IsFatal: !isHealthy, + ErrorCode: []string{"94"}, + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: gpuUUID}}, + GeneratedTimestamp: timestamppb.New(createdAt.Add(-time.Second)), + }) +} + +func persistedRecovery( + createdAt time.Time, + source datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, +) datamodels.HealthEventWithStatus { + event := proto.Clone(source.HealthEvent).(*protos.HealthEvent) + event.Agent = agentName + event.CheckName = rule.Name + event.IsHealthy = true + event.IsFatal = false + event.ErrorCode = nil + event.RecommendedAction = protos.RecommendedAction_NONE + if rule.Recovery.Scope == config.RecoveryScopeNode { + event.EntitiesImpacted = nil + } + + return storedEvent(createdAt, event) +} + +func persistedRecoveryForIdentity( + createdAt time.Time, + source datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, + identity recoveryIdentity, +) datamodels.HealthEventWithStatus { + event := persistedRecovery(createdAt, source, rule) + event.HealthEvent.EntitiesImpacted = identity.entities + + return event +} + +func persistedFault( + createdAt time.Time, + source datamodels.HealthEventWithStatus, + rule config.HealthEventsAnalyzerRule, + entities []*protos.Entity, +) datamodels.HealthEventWithStatus { + event := proto.Clone(source.HealthEvent).(*protos.HealthEvent) + event.Agent = agentName + event.CheckName = rule.Name + event.IsHealthy = false + event.IsFatal = true + event.EntitiesImpacted = entities + + return storedEvent(createdAt, event) +} + +func newRecoveryReconciler( + rule config.HealthEventsAnalyzerRule, + database client.DatabaseClient, + platform protos.PlatformConnectorClient, +) *Reconciler { + return &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{Rules: []config.HealthEventsAnalyzerRule{rule}}, + Publisher: publisher.NewPublisher(platform, protos.ProcessingStrategy_EXECUTE_REMEDIATION), + }, + databaseClient: database, + } +} + +func TestRecoveryIdentityUsesConfiguredEntitySet(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + event := &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{ + {EntityType: "GPU", EntityValue: "0"}, + {EntityType: "GPU_UUID", EntityValue: "GPU-a"}, + {EntityType: "GPU_UUID", EntityValue: "GPU-a"}, + }, + } + + identity, ok := recoveryIdentityForEvent(rule, event) + require.True(t, ok) + require.Equal(t, "node-a|8:GPU_UUID=5:GPU-a", identity.key) + require.Len(t, identity.entities, 1) + require.Equal(t, "GPU_UUID", identity.entities[0].EntityType) + require.Equal(t, "GPU-a", identity.entities[0].EntityValue) + + event.EntitiesImpacted = []*protos.Entity{ + {EntityType: "GPU_UUID", EntityValue: "GPU-a"}, + {EntityType: "GPU_UUID", EntityValue: "GPU-b"}, + } + _, ok = recoveryIdentityForEvent(rule, event) + require.False(t, ok) + + event.EntitiesImpacted = []*protos.Entity{{EntityType: "GPU", EntityValue: "0"}} + _, ok = recoveryIdentityForEvent(rule, event) + require.False(t, ok) + + rule.Recovery.EntityTypes = []string{"GPU_UUID", "PCI"} + event.EntitiesImpacted = []*protos.Entity{ + nil, + {EntityType: "GPU_UUID"}, + {EntityType: "GPU_UUID", EntityValue: "GPU-a"}, + {EntityType: "PCI", EntityValue: "0000:b4:00.0"}, + } + identity, ok = recoveryIdentityForEvent(rule, event) + require.True(t, ok) + require.Equal(t, "GPU_UUID", identity.entities[0].EntityType) + require.Equal(t, "PCI", identity.entities[1].EntityType) +} + +func TestRecoveryIdentityCanBeReadAfterFullDocumentDecodeFails(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + jsonDocument := []byte(`{ + "healthevent": { + "nodeName": "node-a", + "entitiesImpacted": [{"entityType": "GPU_UUID", "entityValue": "GPU-a"}], + "errorCode": {"malformed": true} + } + }`) + bsonDocument, err := bson.Marshal(bson.M{ + "healthevent": bson.M{ + "nodename": "node-a", + "entitiesimpacted": bson.A{ + bson.M{"entitytype": "GPU_UUID", "entityvalue": "GPU-a"}, + }, + "errorcode": bson.M{"malformed": true}, + }, + }) + require.NoError(t, err) + + for name, decode := range map[string]func(any) error{ + "json": func(value any) error { return json.Unmarshal(jsonDocument, value) }, + "bson": func(value any) error { return bson.Unmarshal(bsonDocument, value) }, + } { + t.Run(name, func(t *testing.T) { + cursor := &rawRecoveryIdentityCursor{decode: decode} + var event datamodels.HealthEventWithStatus + require.Error(t, cursor.Decode(&event)) + + identity, decoded, ok := recoveryIdentityFromCurrentDocument(cursor, rule) + require.True(t, decoded) + require.True(t, ok) + require.Equal(t, "node-a|8:GPU_UUID=5:GPU-a", identity.key) + }) + } +} + +func TestRecoveryIdentityAllowsEntityOrNodeWideSource(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + + exact, nodeWide, ok := recoveryIdentityForSource(rule, recoverySource(time.Now(), "GPU-a").HealthEvent) + require.True(t, ok) + require.False(t, nodeWide) + require.Equal(t, "node-a|8:GPU_UUID=5:GPU-a", exact.key) + + node, nodeWide, ok := recoveryIdentityForSource(rule, nodeWideRecoverySource(time.Now()).HealthEvent) + require.True(t, ok) + require.True(t, nodeWide) + require.Equal(t, "node-a|*", node.key) + + partial := nodeWideRecoverySource(time.Now()).HealthEvent + partial.EntitiesImpacted = []*protos.Entity{{EntityType: "PCI", EntityValue: "0000:b4:00.0"}} + _, _, ok = recoveryIdentityForSource(rule, partial) + require.False(t, ok) +} + +func TestHandleRecoveryEventsRejectsNonRecoveryInput(t *testing.T) { + reconciler := &Reconciler{} + + for name, event := range map[string]*datamodels.HealthEventWithStatus{ + "nil": nil, + "missing event": {}, + "unhealthy": {HealthEvent: &protos.HealthEvent{IsHealthy: false}}, + } { + t.Run(name, func(t *testing.T) { + published, err := reconciler.handleRecoveryEvents(context.Background(), event) + require.NoError(t, err) + require.False(t, published) + }) + } +} + +func TestRecoverySourceMatchesConfiguredSource(t *testing.T) { + mapping := recoveryRule(config.RecoveryScopeEntity).Recovery + event := recoverySource(time.Now(), "GPU-a").HealthEvent + require.True(t, recoverySourceMatches(mapping, event)) + + withoutCodeFilter := *mapping + withoutCodeFilter.SourceErrorCodes = nil + healthyReset := proto.Clone(event).(*protos.HealthEvent) + healthyReset.ErrorCode = nil + require.True(t, recoverySourceMatches(&withoutCodeFilter, healthyReset)) + + for name, mutate := range map[string]func(*protos.HealthEvent){ + "unhealthy": func(event *protos.HealthEvent) { event.IsHealthy = false }, + "wrong agent": func(event *protos.HealthEvent) { event.Agent = "other-monitor" }, + "wrong check": func(event *protos.HealthEvent) { event.CheckName = "OtherCheck" }, + "wrong code": func(event *protos.HealthEvent) { event.ErrorCode = []string{"13"} }, + "missing code": func(event *protos.HealthEvent) { event.ErrorCode = nil }, + } { + t.Run(name, func(t *testing.T) { + candidate := proto.Clone(event).(*protos.HealthEvent) + mutate(candidate) + require.False(t, recoverySourceMatches(mapping, candidate)) + }) + } +} + +func TestRecoveryLookupFilterUsesProviderSchema(t *testing.T) { + mongo := (&Reconciler{}).recoveryLookupFilter("monitor", "check", "node") + require.Equal(t, map[string]any{ + "healthevent.agent": "monitor", + "healthevent.checkname": "check", + "healthevent.nodename": "node", + }, mongo) + + postgres := (&Reconciler{provider: datastore.ProviderPostgreSQL}). + recoveryLookupFilter("monitor", "check", "node") + require.Equal(t, map[string]any{ + "healthevent.agent": "monitor", + "event_type": "check", + "node_name": "node", + }, postgres) +} + +func TestHandleEventPublishesScopedRecoveryForActiveLegacyFault(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + recovery := recoverySource(now, "GPU-target") + + // The latest event for another GPU must not hide the older active event for + // the recovery scope. Neither event needs feature-specific metadata. + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor( + derivedEvent(now.Add(-time.Minute), false, "GPU-other"), + derivedEvent(now.Add(-2*time.Minute), false, "GPU-target"), + ), nil, + ).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor(), nil, + ).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor(persistedRecovery(now.Add(time.Second), recovery, rule)), nil, + ).Once() + + var published *protos.HealthEvent + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + published = proto.Clone(args.Get(1).(*protos.HealthEvents).Events[0]).(*protos.HealthEvent) + }). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.True(t, didPublish) + require.NotNil(t, published) + require.True(t, published.IsHealthy) + require.False(t, published.IsFatal) + require.Equal(t, protos.RecommendedAction_NONE, published.RecommendedAction) + require.Equal(t, rule.Name, published.CheckName) + require.Equal(t, []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + published.EntitiesImpacted) + require.Empty(t, published.ErrorCode) + database.AssertNotCalled(t, "Aggregate", mock.Anything, mock.Anything) + database.AssertExpectations(t) + platform.AssertExpectations(t) +} + +func TestHandleEventPublishesNodeScopedRecovery(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeNode) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + recovery := recoverySource(now, "GPU-target") + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(derivedEvent(now.Add(-time.Minute), false, "GPU-other")), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(persistedRecovery(now.Add(time.Second), recovery, rule)), nil). + Once() + + var published *protos.HealthEvent + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + published = proto.Clone(args.Get(1).(*protos.HealthEvents).Events[0]).(*protos.HealthEvent) + }). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.True(t, didPublish) + require.NotNil(t, published) + require.Empty(t, published.EntitiesImpacted) + database.AssertExpectations(t) + platform.AssertExpectations(t) +} + +func TestNodeWideRecoveryClearsAllActiveEntityConditions(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + recovery := nodeWideRecoverySource(now) + + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor( + derivedEvent(now.Add(-4*time.Minute), false, "GPU-b"), + derivedEvent(now.Add(-3*time.Minute), false, "GPU-a"), + derivedEvent(now.Add(-2*time.Minute), true, "GPU-b"), + derivedEvent(now.Add(-time.Minute), false, "GPU-c"), + ), nil, + ).Once() + + for index, gpu := range []string{"GPU-a", "GPU-c"} { + identity, ok := recoveryIdentityForEvent(rule, &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: gpu}}, + }) + require.True(t, ok) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor(), nil, + ).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor(persistedRecoveryForIdentity( + now.Add(time.Duration(index+1)*time.Second), recovery, rule, identity, + )), nil, + ).Once() + } + + publishedGPUs := make([]string, 0, 2) + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + event := args.Get(1).(*protos.HealthEvents).Events[0] + publishedGPUs = append(publishedGPUs, event.EntitiesImpacted[0].EntityValue) + }). + Return(&emptypb.Empty{}, nil). + Twice() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.True(t, didPublish) + require.Equal(t, []string{"GPU-a", "GPU-c"}, publishedGPUs) + platform.AssertExpectations(t) + database.AssertExpectations(t) +} + +func TestRecoveryWaitsForPersistedOutputBeforeReturning(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Millisecond + recovery := recoverySource(now, "GPU-target") + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(derivedEvent(now.Add(-time.Minute), false, "GPU-target")), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(persistedRecovery(now.Add(time.Second), recovery, rule)), nil). + Once() + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.True(t, didPublish) + + database.AssertNumberOfCalls(t, "Find", 4) + platform.AssertNumberOfCalls(t, "HealthEventOccurredV1", 1) +} + +func TestRecoveryRepublishesWhenAcceptedOutputIsNotStored(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Millisecond + reconciler.recoveryRepublish = time.Millisecond + recovery := recoverySource(now, "GPU-target") + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(derivedEvent(now.Add(-time.Minute), false, "GPU-target")), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(persistedRecovery(now.Add(time.Second), recovery, rule)), nil). + Once() + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Return(&emptypb.Empty{}, nil). + Twice() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.True(t, didPublish) + database.AssertNumberOfCalls(t, "Find", 4) + platform.AssertNumberOfCalls(t, "HealthEventOccurredV1", 2) +} + +func TestRecoveryRetriesFailedEnqueue(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Millisecond + reconciler.recoveryRepublish = time.Millisecond + recovery := recoverySource(now, "GPU-target") + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(derivedEvent(now.Add(-time.Minute), false, "GPU-target")), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(persistedRecovery(now.Add(time.Second), recovery, rule)), nil). + Once() + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Return((*emptypb.Empty)(nil), errors.New("connector unavailable")). + Once() + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.True(t, didPublish) + platform.AssertNumberOfCalls(t, "HealthEventOccurredV1", 2) + database.AssertExpectations(t) +} + +func TestRecoveryCancellationBeforePersistenceReturnsError(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + recovery := recoverySource(now, "GPU-target") + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(derivedEvent(now.Add(-time.Minute), false, "GPU-target")), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + + ctx, cancel := context.WithCancel(context.Background()) + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(mock.Arguments) { cancel() }). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(ctx, &recovery) + require.False(t, didPublish) + require.Error(t, err) + require.True(t, errors.Is(err, context.Canceled)) + database.AssertNumberOfCalls(t, "Find", 2) + platform.AssertNumberOfCalls(t, "HealthEventOccurredV1", 1) +} + +func TestDelayedRecoveryDoesNotClearNewerDerivedFault(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + recovery := recoverySource(now.Add(time.Minute), "GPU-target") + recovery.HealthEvent.GeneratedTimestamp = timestamppb.New(now.Add(-time.Hour)) + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(derivedEvent(now, false, "GPU-target")), nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.False(t, didPublish) + platform.AssertNotCalled(t, "HealthEventOccurredV1", mock.Anything, mock.Anything) + + identity, ok := recoveryIdentityForEvent(rule, recovery.HealthEvent) + require.True(t, ok) + _, found := reconciler.cachedRecoveryBoundary(rule.Name, identity) + require.False(t, found) +} + +func TestPersistedRecoveryMustBelongToCurrentSource(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + source := recoverySource(now, "GPU-target") + old := persistedRecovery(now.Add(-time.Second), source, rule) + newer := persistedRecovery(now.Add(time.Second), source, rule) + + require.False(t, sameRecoverySource(&old, &source)) + require.True(t, sameRecoverySource(&newer, &source)) + + // Current publishers give the derived event a fresh generated timestamp and + // preserve the source timestamp in metadata. + newer.HealthEvent.GeneratedTimestamp = timestamppb.New(now.Add(time.Second)) + newer.HealthEvent.Metadata = map[string]string{ + publisher.SourceGeneratedTimestampMetadataKey: source.HealthEvent.GeneratedTimestamp.AsTime(). + UTC().Format(time.RFC3339Nano), + } + require.True(t, sameRecoverySource(&newer, &source)) + + newer.HealthEvent.Metadata[publisher.SourceGeneratedTimestampMetadataKey] = now.Add(time.Hour).Format(time.RFC3339Nano) + require.False(t, sameRecoverySource(&newer, &source)) + + // Legacy sources without a generated timestamp fall back to datastore order. + source.HealthEvent.GeneratedTimestamp = nil + delete(newer.HealthEvent.Metadata, publisher.SourceGeneratedTimestampMetadataKey) + require.True(t, sameRecoverySource(&newer, &source)) + require.False(t, sameRecoverySource(&old, &source)) +} + +func TestRecoveryDoesNotPublishWithoutActiveDerivedFault(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + + for name, events := range map[string][]datamodels.HealthEventWithStatus{ + "no derived event": nil, + "already healthy": {derivedEvent(time.Now(), true, "GPU-target")}, + } { + t.Run(name, func(t *testing.T) { + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + recovery := recoverySource(time.Now().UTC(), "GPU-target") + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(events...), nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &recovery) + require.NoError(t, err) + require.False(t, didPublish) + platform.AssertNotCalled(t, "HealthEventOccurredV1", mock.Anything, mock.Anything) + + identity, ok := recoveryIdentityForEvent(rule, recovery.HealthEvent) + require.True(t, ok) + boundary, found := reconciler.cachedRecoveryBoundary(rule.Name, identity) + require.True(t, found) + require.Equal(t, recovery.CreatedAt, boundary.createdAt) + }) + } +} + +func TestRecoveryBoundaryTruncatesStoredAndOutOfOrderHistory(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + reconciler := &Reconciler{} + boundaryTime := time.Date(2026, 8, 29, 10, 0, 0, 250_000_000, time.UTC) + boundary := &recoveryBoundary{ + createdAt: boundaryTime, + generated: timestamppb.New(boundaryTime.Add(-time.Second)), + } + event := storedEvent(boundaryTime.Add(time.Minute), &protos.HealthEvent{ + NodeName: "node-a", + }) + + pipeline, err := reconciler.getPipelineStages(rule, event, boundary) + require.NoError(t, err) + match := pipeline[0]["$match"].(map[string]any) + require.Equal(t, map[string]any{"$gt": boundaryTime}, match["createdAt"]) + require.Equal(t, []any{ + map[string]any{ + "$or": []any{ + map[string]any{ + fieldGeneratedTimestamp: map[string]any{"$exists": false}, + }, + map[string]any{ + "$expr": generatedAfterExpression(boundary.generated), + }, + }, + }, + }, match["$and"]) +} + +func TestRecoveryBoundaryLoadsAndCachesLatestMatchingSource(t *testing.T) { + base := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + reconciler := &Reconciler{databaseClient: database} + incoming := &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + } + wrongEntity := recoverySource(base.Add(2*time.Hour), "GPU-other") + matching := recoverySource(base.Add(time.Hour), "GPU-target") + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(wrongEntity, matching), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + + boundary, err := reconciler.recoveryBoundaryForEvent(context.Background(), rule, incoming) + require.NoError(t, err) + require.NotNil(t, boundary) + require.Equal(t, matching.CreatedAt, boundary.createdAt) + require.True(t, proto.Equal(matching.HealthEvent.GeneratedTimestamp, boundary.generated)) + + // The second lookup uses the in-memory candidate established by the first, + // but still rechecks that candidate against this identity's derived state. + cached, err := reconciler.recoveryBoundaryForEvent(context.Background(), rule, incoming) + require.NoError(t, err) + require.Equal(t, boundary, cached) + database.AssertNumberOfCalls(t, "Find", 3) +} + +func TestNodeWideRecoveryIsBoundaryForEveryEntity(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + database := new(mockDatabaseClient) + reconciler := &Reconciler{databaseClient: database} + source := nodeWideRecoverySource(now) + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(source), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + + first := &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-a"}}, + } + boundary, err := reconciler.recoveryBoundaryForEvent(context.Background(), rule, first) + require.NoError(t, err) + require.NotNil(t, boundary) + require.Equal(t, now, boundary.createdAt) + + second := &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-b"}}, + } + boundary, err = reconciler.recoveryBoundaryForEvent(context.Background(), rule, second) + require.NoError(t, err) + require.NotNil(t, boundary) + require.Equal(t, now, boundary.createdAt) + database.AssertNumberOfCalls(t, "Find", 3) +} + +func TestNodeWideBoundaryDoesNotBypassGuardForUnrecoveredSibling(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + source := nodeWideRecoverySource(now) + fault := derivedEvent(now.Add(-time.Minute), false, "GPU-b") + events := map[string]*protos.HealthEvent{ + "GPU-a": { + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-a"}}, + }, + "GPU-b": { + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-b"}}, + }, + } + + for _, order := range [][]string{{"GPU-a", "GPU-b"}, {"GPU-b", "GPU-a"}} { + name := order[0] + "_then_" + order[1] + t.Run(name, func(t *testing.T) { + database := new(mockDatabaseClient) + if order[0] == "GPU-a" { + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(source), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(fault), nil).Once() + } else { + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(source), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(fault), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(source), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil).Once() + } + + reconciler := &Reconciler{databaseClient: database} + boundaries := make(map[string]*recoveryBoundary, len(order)) + for _, gpu := range order { + boundary, err := reconciler.recoveryBoundaryForEvent( + context.Background(), rule, events[gpu], + ) + require.NoError(t, err) + boundaries[gpu] = boundary + } + + require.NotNil(t, boundaries["GPU-a"]) + require.Nil(t, boundaries["GPU-b"]) + database.AssertExpectations(t) + }) + } +} + +func TestRecoveryBoundaryRequiresRecoveredDerivedStateAfterRestart(t *testing.T) { + base := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + incoming := &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + } + source := recoverySource(base, "GPU-target") + + for name, derived := range map[string]datamodels.HealthEventWithStatus{ + "active fault": derivedEvent(base.Add(-time.Minute), false, "GPU-target"), + "persisted recovery": persistedRecovery(base.Add(time.Second), source, rule), + } { + t.Run(name, func(t *testing.T) { + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(source), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(derived), nil). + Once() + reconciler := &Reconciler{databaseClient: database} + + boundary, err := reconciler.recoveryBoundaryForEvent(context.Background(), rule, incoming) + require.NoError(t, err) + if name == "active fault" { + require.Nil(t, boundary) + identity, ok := recoveryIdentityForEvent(rule, incoming) + require.True(t, ok) + _, found := reconciler.cachedRecoveryBoundary(rule.Name, identity) + require.False(t, found) + return + } + + require.NotNil(t, boundary) + require.Equal(t, source.CreatedAt, boundary.createdAt) + }) + } +} + +func TestRecoveryBoundaryAdvancesOnlyAfterDurableRecovery(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + source := recoverySource(now, "GPU-target") + identity, ok := recoveryIdentityForEvent(rule, source.HealthEvent) + require.True(t, ok) + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(&healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(now, true, "GPU-target")}, + pos: -1, + decodeErr: errors.New("malformed persisted recovery"), + }, nil).Once() + reconciler := &Reconciler{databaseClient: database} + + _, err := reconciler.publishRecoveryTargets( + context.Background(), &source, rule, + []recoveryTarget{{ + identity: identity, + state: derivedState{ + boundary: recoveryBoundary{createdAt: now.Add(-time.Minute)}, + isHealthy: false, + }, + }}, + boundaryFromEvent(&source), false, + ) + require.Error(t, err) + require.True(t, client.IsPermanentError(err)) + _, found := reconciler.cachedRecoveryBoundary(rule.Name, identity) + require.False(t, found) +} + +func TestNodeWideRecoveryBoundaryAdvancesOnlyAfterAllRecoveries(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + source := nodeWideRecoverySource(now) + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor(derivedEvent(now.Add(-time.Minute), false, "GPU-target")), nil, + ).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(&healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(now, true, "GPU-target")}, + pos: -1, + decodeErr: errors.New("malformed persisted recovery"), + }, nil).Once() + reconciler := &Reconciler{databaseClient: database} + + _, err := reconciler.handleRecoveryRule(context.Background(), &source, rule) + require.Error(t, err) + require.True(t, client.IsPermanentError(err)) + _, found := reconciler.cachedRecoveryBoundary(rule.Name, nodeRecoveryIdentity("node-a")) + require.False(t, found) +} + +func TestRecoveryBoundaryCacheUsesGenerationOrder(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + reconciler := &Reconciler{} + identity := recoveryIdentity{key: "node-a|GPU-a", nodeName: "node-a"} + base := time.Now().UTC() + + reconciler.rememberRecoveryBoundary(rule.Name, identity, recoveryBoundary{ + createdAt: base.Add(time.Hour), + generated: timestamppb.New(base), + }) + reconciler.rememberRecoveryBoundary(rule.Name, identity, recoveryBoundary{ + createdAt: base, + generated: timestamppb.New(base.Add(2 * time.Hour)), + }) + + cached, found := reconciler.cachedRecoveryBoundary(rule.Name, identity) + require.True(t, found) + require.Equal(t, base.Add(2*time.Hour), cached.generated.AsTime()) + + // A delayed older recovery must not replace the newer generation merely + // because it was stored later. + reconciler.rememberRecoveryBoundary(rule.Name, identity, recoveryBoundary{ + createdAt: base.Add(3 * time.Hour), + generated: timestamppb.New(base.Add(-time.Hour)), + }) + cached, found = reconciler.cachedRecoveryBoundary(rule.Name, identity) + require.True(t, found) + require.Equal(t, base.Add(2*time.Hour), cached.generated.AsTime()) +} + +func TestFindLatestMatchingEventUsesGenerationOrder(t *testing.T) { + base := time.Now().UTC() + staleButStoredLater := derivedEvent(base.Add(2*time.Hour), false, "GPU-target") + staleButStoredLater.HealthEvent.GeneratedTimestamp = timestamppb.New(base) + newerButStoredEarlier := derivedEvent(base.Add(time.Hour), true, "GPU-target") + newerButStoredEarlier.HealthEvent.GeneratedTimestamp = timestamppb.New(base.Add(3 * time.Hour)) + + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(staleButStoredLater, newerButStoredEarlier), nil). + Once() + reconciler := &Reconciler{databaseClient: database} + + latest, err := reconciler.findLatestMatchingEvent( + context.Background(), + nil, + nil, + "test-rule", + "test", + map[string]any{"event_type": "RepeatedXID94OnSameGPU"}, + func(*datamodels.HealthEventWithStatus) bool { return true }, + ) + require.NoError(t, err) + require.NotNil(t, latest) + require.True(t, latest.HealthEvent.IsHealthy) + require.Equal(t, base.Add(3*time.Hour), latest.HealthEvent.GeneratedTimestamp.AsTime()) +} + +func TestRecoveryBoundaryQueryFailurePreventsRuleEvaluation(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + incoming := storedEvent(now, &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + NodeName: "node-a", + ErrorCode: []string{"94"}, + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + GeneratedTimestamp: timestamppb.New(now), + }) + + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return((*healthEventCursor)(nil), errors.New("query failed")). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &incoming) + require.False(t, didPublish) + require.ErrorContains(t, err, "find recovery boundary") + database.AssertNotCalled(t, "Aggregate", mock.Anything, mock.Anything) + platform.AssertNotCalled(t, "HealthEventOccurredV1", mock.Anything, mock.Anything) +} + +func TestActiveDerivedFaultDoesNotSuppressRecurringFault(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + incoming := storedEvent(now, &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + NodeName: "node-a", + ErrorCode: []string{"94"}, + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + GeneratedTimestamp: timestamppb.New(now), + }) + + // A legacy active derived fault must not suppress a new matching source + // event. Manual condition cleanup is not represented in event history. + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + aggregateCursor, _ := createMockCursor([]map[string]any{{"ruleMatched": true}}) + database.On("Aggregate", mock.Anything, mock.Anything).Return(aggregateCursor, nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor( + derivedEvent(now.Add(-time.Minute), false, "GPU-target"), + ), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor( + derivedEvent(now.Add(-time.Minute), false, "GPU-target"), + persistedFault(now.Add(time.Second), incoming, rule, + []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}), + ), nil). + Once() + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &incoming) + require.NoError(t, err) + require.True(t, didPublish) + platform.AssertExpectations(t) +} + +func TestDerivedFaultWaitsForPersistenceAndUsesRecoveryScope(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Millisecond + reconciler.recoveryRepublish = time.Millisecond + incoming := storedEvent(now, &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + NodeName: "node-a", + ErrorCode: []string{"94"}, + EntitiesImpacted: []*protos.Entity{ + {EntityType: "PCI", EntityValue: "0000:b4:00.0"}, + {EntityType: "GPU_UUID", EntityValue: "GPU-target"}, + }, + GeneratedTimestamp: timestamppb.New(now), + }) + + identity, ok := recoveryIdentityForEvent(rule, incoming.HealthEvent) + require.True(t, ok) + reconciler.rememberRecoveryBoundary(rule.Name, identity, recoveryBoundary{ + createdAt: now.Add(-time.Minute), + generated: timestamppb.New(now.Add(-time.Minute)), + }) + reconciler.rememberDerivedState(rule.Name, identity, derivedState{ + boundary: recoveryBoundary{createdAt: now.Add(-time.Minute)}, + isHealthy: true, + }) + + aggregate, _ := createMockCursor([]map[string]any{{"ruleMatched": true}}) + database.On("Aggregate", mock.Anything, mock.Anything).Return(aggregate, nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil). + Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(persistedFault( + now.Add(time.Second), incoming, rule, + []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + )), nil). + Once() + + var published *protos.HealthEvent + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + published = proto.Clone(args.Get(1).(*protos.HealthEvents).Events[0]).(*protos.HealthEvent) + }). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &incoming) + require.NoError(t, err) + require.True(t, didPublish) + platform.AssertNumberOfCalls(t, "HealthEventOccurredV1", 1) + require.Equal(t, []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + published.EntitiesImpacted) + database.AssertExpectations(t) +} + +func TestRecoveryEnabledRulePreservesFaultWithoutConfiguredEntity(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + incoming := storedEvent(now, &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + NodeName: "node-a", + ErrorCode: []string{"94"}, + EntitiesImpacted: []*protos.Entity{{EntityType: "PCI", EntityValue: "0000:b4:00.0"}}, + GeneratedTimestamp: timestamppb.New(now), + }) + + aggregate, _ := createMockCursor([]map[string]any{{"ruleMatched": true}}) + database.On("Aggregate", mock.Anything, mock.Anything).Return(aggregate, nil).Once() + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Return(&emptypb.Empty{}, nil). + Once() + + didPublish, err := reconciler.handleEvent(context.Background(), &incoming) + require.NoError(t, err) + require.True(t, didPublish) + platform.AssertExpectations(t) +} + +func TestRecoveryRuleSkipsDisabledAndIncompleteScope(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + recovery := recoverySource(time.Now().UTC(), "GPU-target") + reconciler := newRecoveryReconciler(rule, new(mockDatabaseClient), new(mockPublisher)) + + disabled := rule + disabled.EvaluateRule = false + published, err := reconciler.handleRecoveryRule(context.Background(), &recovery, disabled) + require.NoError(t, err) + require.False(t, published) + + recovery.HealthEvent.EntitiesImpacted = []*protos.Entity{{ + EntityType: "PCI", EntityValue: "0000:b4:00.0", + }} + published, err = reconciler.handleRecoveryRule(context.Background(), &recovery, rule) + require.NoError(t, err) + require.False(t, published) +} + +func TestRecoveryStateQueriesPropagateProviderErrors(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + recovery := recoverySource(time.Now().UTC(), "GPU-target") + identity, ok := recoveryIdentityForEvent(rule, recovery.HealthEvent) + require.True(t, ok) + + for name, call := range map[string]func(*Reconciler) error{ + "handle recovery": func(reconciler *Reconciler) error { + _, err := reconciler.handleRecoveryRule(context.Background(), &recovery, rule) + return err + }, + "current derived state": func(reconciler *Reconciler) error { + _, _, err := reconciler.currentDerivedState(context.Background(), rule, identity) + return err + }, + } { + t.Run(name, func(t *testing.T) { + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return((*healthEventCursor)(nil), errors.New("provider unavailable")).Once() + reconciler := newRecoveryReconciler(rule, database, new(mockPublisher)) + require.ErrorContains(t, call(reconciler), "provider unavailable") + }) + } +} + +func TestRecoveryBoundaryReportsInvalidStoredIdentity(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + reconciler := &Reconciler{databaseClient: database} + metric := recoveryStoredDocumentDecodeErrorsTotal.WithLabelValues( + rule.Name, "recovery_source", "invalid_identity", + ) + before := testutil.ToFloat64(metric) + + boundary, err := reconciler.recoveryBoundaryForEvent(context.Background(), rule, + &protos.HealthEvent{NodeName: "node-a"}) + require.NoError(t, err) + require.Nil(t, boundary) + + incoming := &protos.HealthEvent{ + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + } + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return( + newHealthEventCursor( + recoverySource(time.Now().UTC(), "GPU-other"), + storedEvent(time.Now().UTC(), &protos.HealthEvent{ + Agent: "syslog-health-monitor", CheckName: "SysLogsXIDError", IsHealthy: true, + NodeName: "node-a", + EntitiesImpacted: []*protos.Entity{{EntityType: "PCI", EntityValue: "0000:b4:00.0"}}, + }), + storedEvent(time.Now().UTC(), nil), + ), nil, + ).Once() + boundary, err = reconciler.recoveryBoundaryForEvent(context.Background(), rule, incoming) + require.NoError(t, err) + require.Nil(t, boundary) + require.Equal(t, before+1, testutil.ToFloat64(metric)) + database.AssertNumberOfCalls(t, "Find", 1) +} + +func TestFindLatestMatchingEventReportsCursorFailures(t *testing.T) { + for name, cursor := range map[string]*healthEventCursor{ + "decode": { + events: []datamodels.HealthEventWithStatus{derivedEvent(time.Now(), false, "GPU-a")}, + pos: -1, + decodeErr: errors.New("decode failed"), + }, + "iteration": {pos: -1, err: errors.New("iteration failed")}, + } { + t.Run(name, func(t *testing.T) { + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(cursor, nil).Once() + reconciler := &Reconciler{databaseClient: database} + _, err := reconciler.findLatestMatchingEvent( + context.Background(), nil, nil, "test-rule", "test", map[string]any{}, + func(*datamodels.HealthEventWithStatus) bool { return true }, + ) + require.Error(t, err) + require.Equal(t, name == "decode", client.IsPermanentError(err)) + }) + } +} + +func TestOutOfScopeStoredDecodeFailureIsReportedWithoutAbortingLookup(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Name = "out-of-scope-reporting-rule" + target := mustRecoveryIdentity(t, rule, derivedEvent(time.Now(), false, "GPU-target").HealthEvent) + cursor := &healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(time.Now(), false, "GPU-other")}, + pos: -1, + decodeErrs: map[int]error{0: errors.New("malformed sibling document")}, + } + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(cursor, nil).Once() + reconciler := &Reconciler{databaseClient: database} + metric := recoveryStoredDocumentDecodeErrorsTotal.WithLabelValues( + rule.Name, "derived_state", "malformed", + ) + before := testutil.ToFloat64(metric) + + latest, err := reconciler.findLatestMatchingEvent( + context.Background(), &rule, &target, rule.Name, "derived_state", map[string]any{}, + func(*datamodels.HealthEventWithStatus) bool { return true }, + ) + + require.NoError(t, err) + require.Nil(t, latest) + require.Equal(t, before+1, testutil.ToFloat64(metric)) + database.AssertExpectations(t) +} + +func TestRecoveryDecodeClassificationPreservesTransientFailures(t *testing.T) { + for name, err := range map[string]error{ + "deadline": context.DeadlineExceeded, + "bad connection": driver.ErrBadConn, + } { + t.Run(name, func(t *testing.T) { + classified := classifyRecoveryDecodeError(context.Background(), err) + require.False(t, client.IsPermanentError(classified)) + }) + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + classified := classifyRecoveryDecodeError(canceled, errors.New("scan interrupted")) + require.False(t, client.IsPermanentError(classified)) + require.ErrorIs(t, classified, context.Canceled) + + require.True(t, client.IsPermanentError( + classifyRecoveryDecodeError(context.Background(), errors.New("malformed document")), + )) +} + +func TestRecoveryDecodeCallSitesPreserveTransientFailures(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + + for name, call := range map[string]func(*Reconciler) error{ + "node derived states": func(reconciler *Reconciler) error { + _, err := reconciler.currentDerivedStatesForNode(context.Background(), rule, "node-a") + return err + }, + "latest matching event": func(reconciler *Reconciler) error { + _, err := reconciler.findLatestMatchingEvent( + context.Background(), nil, nil, rule.Name, "test", map[string]any{}, + func(*datamodels.HealthEventWithStatus) bool { return true }, + ) + return err + }, + } { + t.Run(name, func(t *testing.T) { + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(&healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(time.Now(), false, "GPU-a")}, + pos: -1, + decodeErrs: map[int]error{0: driver.ErrBadConn}, + }, nil).Once() + reconciler := &Reconciler{databaseClient: database} + + err := call(reconciler) + require.Error(t, err) + require.False(t, client.IsPermanentError(err), err) + }) + } +} + +func TestCurrentDerivedStatesMarksDecodeFailurePermanent(t *testing.T) { + cursor := &healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(time.Now(), false, "GPU-a")}, + pos: -1, + decodeErr: errors.New("decode failed"), + } + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(cursor, nil).Once() + reconciler := &Reconciler{databaseClient: database} + + _, err := reconciler.currentDerivedStatesForNode( + context.Background(), recoveryRule(config.RecoveryScopeEntity), "node-a", + ) + require.Error(t, err) + require.True(t, client.IsPermanentError(err)) +} + +func TestHandleEventAllowsCheckpointAfterStoredDocumentDecodeFailure(t *testing.T) { + rule := recoveryRule(config.RecoveryScopeEntity) + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(&healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(time.Now(), false, "GPU-a")}, + pos: -1, + decodeErr: errors.New("decode failed"), + }, nil).Once() + reconciler := newRecoveryReconciler(rule, database, new(mockPublisher)) + recovery := recoverySource(time.Now(), "GPU-a") + + published, err := reconciler.handleEvent(context.Background(), &recovery) + require.False(t, published) + require.NoError(t, err) + _, cached := reconciler.cachedRecoveryBoundary(rule.Name, mustRecoveryIdentity(t, rule, recovery.HealthEvent)) + require.False(t, cached) +} + +func TestMalformedStoredDocumentRecoversUnaffectedTarget(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + source := nodeWideRecoverySource(now) + goodIdentity, ok := recoveryIdentityForEvent(rule, derivedEvent(now, false, "GPU-good").HealthEvent) + require.True(t, ok) + poisoned := derivedEvent(now.Add(-2*time.Minute), false, "GPU-poisoned") + poisonedHealthy := derivedEvent(now.Add(-3*time.Minute), true, "GPU-poisoned") + good := derivedEvent(now.Add(-time.Minute), false, "GPU-good") + database := newRecoveryDocumentSetDatabase( + []datamodels.HealthEventWithStatus{poisoned, poisonedHealthy, good}, + map[int]error{0: errors.New("malformed stored document")}, + ) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Millisecond + + metric := recoveryStoredDocumentDecodeErrorsTotal.WithLabelValues( + rule.Name, "node_derived_states", "malformed", + ) + before := testutil.ToFloat64(metric) + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + event := proto.Clone(args.Get(1).(*protos.HealthEvents).Events[0]).(*protos.HealthEvent) + database.append(storedEvent(now.Add(time.Second), event)) + }). + Return(&emptypb.Empty{}, nil).Once() + + published, err := reconciler.handleRecoveryEvents(context.Background(), &source) + require.True(t, published) + require.NoError(t, err) + require.Equal(t, before+1, testutil.ToFloat64(metric)) + _, nodeBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, nodeRecoveryIdentity("node-a")) + require.False(t, nodeBoundaryCached) + poisonedIdentity := mustRecoveryIdentity(t, rule, poisoned.HealthEvent) + _, poisonedBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, poisonedIdentity) + require.False(t, poisonedBoundaryCached) + _, goodBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, goodIdentity) + require.True(t, goodBoundaryCached) + require.GreaterOrEqual(t, database.findCalls, 3) + platform.AssertExpectations(t) +} + +func TestInvalidStoredIdentityBlocksNodeBoundaryAndIsCounted(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + source := nodeWideRecoverySource(now) + invalid := derivedEvent(now.Add(-time.Minute), false, "GPU-invalid") + invalid.HealthEvent.EntitiesImpacted = nil + good := derivedEvent(now.Add(-30*time.Second), false, "GPU-good") + goodIdentity := mustRecoveryIdentity(t, rule, good.HealthEvent) + database := newRecoveryDocumentSetDatabase( + []datamodels.HealthEventWithStatus{invalid, good}, nil, + ) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Millisecond + metric := recoveryStoredDocumentDecodeErrorsTotal.WithLabelValues( + rule.Name, "node_derived_states", "invalid_identity", + ) + before := testutil.ToFloat64(metric) + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + event := proto.Clone(args.Get(1).(*protos.HealthEvents).Events[0]).(*protos.HealthEvent) + database.append(storedEvent(now.Add(time.Second), event)) + }). + Return(&emptypb.Empty{}, nil).Once() + + published, err := reconciler.handleRecoveryEvents(context.Background(), &source) + require.NoError(t, err) + require.True(t, published) + require.Equal(t, before+1, testutil.ToFloat64(metric)) + _, nodeBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, nodeRecoveryIdentity("node-a")) + require.False(t, nodeBoundaryCached) + _, goodBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, goodIdentity) + require.True(t, goodBoundaryCached) + platform.AssertExpectations(t) +} + +func TestUnreadableStoredIdentityWithholdsEveryNodeTarget(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + source := nodeWideRecoverySource(now) + corrupt := derivedEvent(now.Add(-time.Minute), false, "GPU-corrupt") + good := derivedEvent(now.Add(-30*time.Second), false, "GPU-good") + database := newRecoveryDocumentSetDatabase( + []datamodels.HealthEventWithStatus{corrupt, good}, + map[int]error{0: errors.New("malformed stored document")}, + ) + database.identityDecodeErrs = map[int]error{0: errors.New("unreadable recovery identity")} + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + metric := recoveryStoredDocumentDecodeErrorsTotal.WithLabelValues( + rule.Name, "node_derived_states", "malformed", + ) + before := testutil.ToFloat64(metric) + + published, err := reconciler.handleRecoveryEvents(context.Background(), &source) + require.NoError(t, err) + require.False(t, published) + require.Equal(t, before+1, testutil.ToFloat64(metric)) + _, nodeBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, nodeRecoveryIdentity("node-a")) + require.False(t, nodeBoundaryCached) + _, goodBoundaryCached := reconciler.cachedRecoveryBoundary( + rule.Name, mustRecoveryIdentity(t, rule, good.HealthEvent), + ) + require.False(t, goodBoundaryCached) + platform.AssertNotCalled(t, "HealthEventOccurredV1", mock.Anything, mock.Anything) +} + +func TestTransientNodeScanPublishesCleanTargetsBeforeReplay(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + source := nodeWideRecoverySource(now) + transient := derivedEvent(now.Add(-time.Minute), false, "GPU-transient") + good := derivedEvent(now.Add(-30*time.Second), false, "GPU-good") + database := newRecoveryDocumentSetDatabase( + []datamodels.HealthEventWithStatus{transient, good}, + map[int]error{0: driver.ErrBadConn}, + ) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Millisecond + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(args mock.Arguments) { + event := proto.Clone(args.Get(1).(*protos.HealthEvents).Events[0]).(*protos.HealthEvent) + database.append(storedEvent(now.Add(time.Second), event)) + }). + Return(&emptypb.Empty{}, nil).Once() + + published, err := reconciler.handleRecoveryEvents(context.Background(), &source) + require.Error(t, err) + require.False(t, client.IsPermanentError(err)) + require.True(t, published) + _, goodBoundaryCached := reconciler.cachedRecoveryBoundary( + rule.Name, mustRecoveryIdentity(t, rule, good.HealthEvent), + ) + require.True(t, goodBoundaryCached) + _, nodeBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, nodeRecoveryIdentity("node-a")) + require.False(t, nodeBoundaryCached) + platform.AssertExpectations(t) +} + +func TestTransientNodeScanRemainsReplayableAfterPermanentTargetFailure(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Recovery.SourceErrorCodes = nil + source := nodeWideRecoverySource(now) + transient := derivedEvent(now.Add(-time.Minute), false, "GPU-transient") + good := derivedEvent(now.Add(-30*time.Second), false, "GPU-good") + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(&healthEventCursor{ + events: []datamodels.HealthEventWithStatus{transient, good}, + pos: -1, + decodeErrs: map[int]error{0: driver.ErrBadConn}, + }, nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(&healthEventCursor{ + events: []datamodels.HealthEventWithStatus{good}, + pos: -1, + decodeErrs: map[int]error{0: errors.New("malformed target document")}, + }, nil).Once() + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + + published, err := reconciler.handleRecoveryEvents(context.Background(), &source) + + require.Error(t, err) + require.False(t, client.IsPermanentError(err)) + require.False(t, published) + _, nodeBoundaryCached := reconciler.cachedRecoveryBoundary(rule.Name, nodeRecoveryIdentity("node-a")) + require.False(t, nodeBoundaryCached) + platform.AssertNotCalled(t, "HealthEventOccurredV1", mock.Anything, mock.Anything) + database.AssertExpectations(t) +} + +func TestStoredDocumentScanCountsKnownTargetWithoutDecodedState(t *testing.T) { + scanErr := &storedDocumentScanError{issues: []*storedDocumentDecodeError{{ + cause: errors.New("malformed stored document"), + classification: "malformed", + identityKey: "node-a|8:GPU_UUID=5:GPU-a", + targetScope: storedDocumentAffectsIdentity, + }}} + + require.Equal(t, 1, scanErr.skippedTargetCount(nil, recoveryIdentity{}, true)) + + target := recoveryIdentity{key: "node-a|8:GPU_UUID=10:GPU-target"} + unreadable := &storedDocumentScanError{issues: []*storedDocumentDecodeError{{ + targetScope: storedDocumentAffectsAllTargets, + }}} + require.Equal(t, 1, unreadable.skippedTargetCount(nil, target, false)) + + invalid := &storedDocumentScanError{issues: []*storedDocumentDecodeError{{ + targetScope: storedDocumentAffectsNoTargets, + }}} + require.True(t, invalid.hasInvalidIdentity()) +} + +func TestStoredDocumentIssueTargetScopes(t *testing.T) { + target := recoveryIdentity{key: "node-a|8:GPU_UUID=10:GPU-target"} + other := recoveryIdentity{key: "node-a|8:GPU_UUID=9:GPU-other"} + + for name, test := range map[string]struct { + issue *storedDocumentDecodeError + affectsGood bool + affectsElse bool + }{ + "unset scope fails closed": { + issue: &storedDocumentDecodeError{}, + affectsGood: true, + affectsElse: true, + }, + "invalid identity affects no target": { + issue: &storedDocumentDecodeError{targetScope: storedDocumentAffectsNoTargets}, + }, + "readable identity affects only its target": { + issue: &storedDocumentDecodeError{ + identityKey: target.key, targetScope: storedDocumentAffectsIdentity, + }, + affectsGood: true, + }, + "unreadable identity affects every target": { + issue: &storedDocumentDecodeError{targetScope: storedDocumentAffectsAllTargets}, + affectsGood: true, + affectsElse: true, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, test.affectsGood, test.issue.affects(target)) + require.Equal(t, test.affectsElse, test.issue.affects(other)) + }) + } +} + +func TestStoredDocumentIssueTrackerSupportsConcurrentContexts(t *testing.T) { + tracker := &storedDocumentIssueTracker{seen: make(map[string]struct{})} + results := make(chan bool, 64) + + var wait sync.WaitGroup + + for range 64 { + wait.Add(1) + + go func() { + defer wait.Done() + results <- tracker.mark("same-issue") + }() + } + + wait.Wait() + close(results) + + firstReports := 0 + for first := range results { + if first { + firstReports++ + } + } + + require.Equal(t, 1, firstReports) +} + +func TestStoredDocumentScanErrorBoundsDetails(t *testing.T) { + scanErr := &storedDocumentScanError{} + for index := range 5 { + scanErr.append(&storedDocumentDecodeError{cause: fmt.Errorf("corrupt row %d", index)}) + } + + message := scanErr.Error() + require.Contains(t, message, "5 stored health event document(s) were incomplete") + require.Contains(t, message, "corrupt row 0") + require.Contains(t, message, "corrupt row 2") + require.NotContains(t, message, "corrupt row 3") + require.Contains(t, message, "and 2 more") +} + +func TestStoredDocumentDecodeMetricIsDeduplicatedAcrossPersistencePolls(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + rule.Name = "deduplicated-reporting-rule" + source := recoverySource(now, "GPU-target") + identity := mustRecoveryIdentity(t, rule, source.HealthEvent) + metric := recoveryStoredDocumentDecodeErrorsTotal.WithLabelValues( + rule.Name, "persisted_derived", "transient", + ) + before := testutil.ToFloat64(metric) + database := new(mockDatabaseClient) + transientCursor := func() *healthEventCursor { + return &healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(now, true, "GPU-target")}, + pos: -1, + decodeErrs: map[int]error{0: driver.ErrBadConn}, + } + } + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(transientCursor(), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(transientCursor(), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(persistedRecovery(now.Add(time.Second), source, rule)), nil).Once() + reconciler := &Reconciler{ + databaseClient: database, + recoveryPoll: time.Millisecond, + recoveryRepublish: time.Hour, + } + publishCalls := 0 + + _, published, err := reconciler.publishDerivedUntilStored( + context.Background(), &source, rule, identity, true, "recovery", + func(context.Context) error { publishCalls++; return nil }, + ) + + require.NoError(t, err) + require.True(t, published) + require.Equal(t, 1, publishCalls) + require.Equal(t, before+1, testutil.ToFloat64(metric)) + database.AssertExpectations(t) +} + +func TestRecoveryOrderingFallbacks(t *testing.T) { + base := time.Now().UTC() + require.True(t, boundaryAfter( + recoveryBoundary{createdAt: base.Add(time.Second)}, + recoveryBoundary{createdAt: base}, + )) + require.False(t, boundaryAfter(recoveryBoundary{}, recoveryBoundary{})) + require.False(t, sameRecoverySource(nil, nil)) + require.False(t, sameRecoverySource( + &datamodels.HealthEventWithStatus{HealthEvent: &protos.HealthEvent{}}, nil, + )) + + invalidTimestamp := ×tamppb.Timestamp{Seconds: 253402300800} + boundary := boundaryFromEvent(&datamodels.HealthEventWithStatus{ + CreatedAt: base, + HealthEvent: &protos.HealthEvent{GeneratedTimestamp: invalidTimestamp}, + }) + require.Nil(t, boundary.generated) +} + +func TestDerivedPersistenceRetriesProviderError(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + source := recoverySource(now, "GPU-target") + identity, ok := recoveryIdentityForEvent(rule, source.HealthEvent) + require.True(t, ok) + database := new(mockDatabaseClient) + reconciler := &Reconciler{ + databaseClient: database, + recoveryPoll: time.Millisecond, + recoveryRepublish: time.Hour, + } + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return((*healthEventCursor)(nil), errors.New("temporary query failure")).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(persistedRecovery(now.Add(time.Second), source, rule)), nil).Once() + + publishCalls := 0 + boundary, published, err := reconciler.publishDerivedUntilStored( + context.Background(), &source, rule, identity, true, "recovery", + func(context.Context) error { publishCalls++; return nil }, + ) + require.NoError(t, err) + require.True(t, published) + require.Equal(t, now.Add(time.Second), boundary.createdAt) + require.Equal(t, 1, publishCalls) +} + +func TestDerivedPersistenceReturnsPermanentLookupError(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + source := recoverySource(now, "GPU-target") + identity, ok := recoveryIdentityForEvent(rule, source.HealthEvent) + require.True(t, ok) + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything).Return(&healthEventCursor{ + events: []datamodels.HealthEventWithStatus{derivedEvent(now, true, "GPU-target")}, + pos: -1, + decodeErr: errors.New("malformed persisted event"), + }, nil).Once() + reconciler := &Reconciler{databaseClient: database} + publishCalls := 0 + + _, published, err := reconciler.publishDerivedUntilStored( + context.Background(), &source, rule, identity, true, "recovery", + func(context.Context) error { publishCalls++; return nil }, + ) + require.Error(t, err) + require.True(t, client.IsPermanentError(err)) + require.False(t, published) + require.Zero(t, publishCalls) +} + +func TestRecoveryEventsContinueAfterPermanentRuleFailure(t *testing.T) { + firstRule := recoveryRule(config.RecoveryScopeEntity) + firstRule.Name = "first-rule" + secondRule := recoveryRule(config.RecoveryScopeEntity) + secondRule.Name = "second-rule" + database := new(mockDatabaseClient) + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return((*healthEventCursor)(nil), client.PermanentError(errors.New("invalid first-rule lookup"))).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil).Once() + reconciler := &Reconciler{ + config: HealthEventsAnalyzerReconcilerConfig{ + HealthEventsAnalyzerRules: &config.TomlConfig{ + Rules: []config.HealthEventsAnalyzerRule{firstRule, secondRule}, + }, + }, + databaseClient: database, + } + source := recoverySource(time.Now(), "GPU-target") + + published, err := reconciler.handleRecoveryEvents(context.Background(), &source) + require.NoError(t, err) + require.False(t, published) + database.AssertExpectations(t) +} + +func TestDerivedPersistenceTimeoutReturnsForReplay(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + source := recoverySource(now, "GPU-target") + identity, ok := recoveryIdentityForEvent(rule, source.HealthEvent) + require.True(t, ok) + database := new(mockDatabaseClient) + reconciler := &Reconciler{ + databaseClient: database, + recoveryPoll: time.Hour, + recoveryTimeout: 5 * time.Millisecond, + } + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil).Once() + + publishCalls := 0 + _, published, err := reconciler.publishDerivedUntilStored( + context.Background(), &source, rule, identity, true, "recovery", + func(context.Context) error { publishCalls++; return nil }, + ) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "timed out waiting for persisted derived recovery") + require.Equal(t, 1, publishCalls) + require.True(t, published) +} + +func TestProcessRulePropagatesPersistenceCancellation(t *testing.T) { + now := time.Now().UTC() + rule := recoveryRule(config.RecoveryScopeEntity) + incoming := storedEvent(now, &protos.HealthEvent{ + Agent: "syslog-health-monitor", + CheckName: "SysLogsXIDError", + NodeName: "node-a", + ErrorCode: []string{"94"}, + EntitiesImpacted: []*protos.Entity{{EntityType: "GPU_UUID", EntityValue: "GPU-target"}}, + GeneratedTimestamp: timestamppb.New(now), + }) + identity, ok := recoveryIdentityForEvent(rule, incoming.HealthEvent) + require.True(t, ok) + + database := new(mockDatabaseClient) + platform := new(mockPublisher) + reconciler := newRecoveryReconciler(rule, database, platform) + reconciler.recoveryPoll = time.Hour + reconciler.rememberRecoveryBoundary(rule.Name, identity, recoveryBoundary{createdAt: now.Add(-time.Minute)}) + aggregate, _ := createMockCursor([]map[string]any{{"ruleMatched": true}}) + database.On("Aggregate", mock.Anything, mock.Anything).Return(aggregate, nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil).Once() + database.On("Find", mock.Anything, mock.Anything, mock.Anything). + Return(newHealthEventCursor(), nil).Once() + ctx, cancel := context.WithCancel(context.Background()) + platform.On("HealthEventOccurredV1", mock.Anything, mock.Anything). + Run(func(mock.Arguments) { cancel() }).Return(&emptypb.Empty{}, nil).Once() + + published, err := reconciler.processRule(ctx, rule, &incoming) + require.False(t, published) + require.ErrorIs(t, err, context.Canceled) +} diff --git a/store-client/pkg/client/event_processor.go b/store-client/pkg/client/event_processor.go index c192a515e..44e93b996 100644 --- a/store-client/pkg/client/event_processor.go +++ b/store-client/pkg/client/event_processor.go @@ -249,6 +249,19 @@ func (p *DefaultEventProcessor) handleSingleEvent(ctx context.Context, event Eve func (p *DefaultEventProcessor) handleProcessingError( ctx context.Context, eventID string, processErr error, token []byte, ) error { + if IsPermanentError(processErr) { + slog.Warn("Checkpointing event after deterministic processing failure", + "eventID", eventID, "error", processErr) + + if markErr := p.markProcessed(ctx, token); markErr != nil { + return newUncheckpointedEventError(fmt.Errorf( + "failed to mark event as processed after permanent error (%w): %w", processErr, markErr, + )) + } + + return processErr + } + if !p.config.MarkProcessedOnError { slog.Error("Event processing failed, NOT marking as processed - will retry on restart", "eventID", eventID, "error", processErr) diff --git a/store-client/pkg/client/interfaces.go b/store-client/pkg/client/interfaces.go index 75d40fb40..300f9dc5e 100644 --- a/store-client/pkg/client/interfaces.go +++ b/store-client/pkg/client/interfaces.go @@ -136,6 +136,8 @@ type SingleResult interface { // Cursor represents a query result cursor type Cursor interface { Next(ctx context.Context) bool + // Decode may be called repeatedly after a successful Next and must decode + // the same current document without advancing the cursor. Decode(v any) error Close(ctx context.Context) error All(ctx context.Context, results any) error diff --git a/store-client/pkg/client/mongodb_client.go b/store-client/pkg/client/mongodb_client.go index 844b419f4..42c2b67bd 100644 --- a/store-client/pkg/client/mongodb_client.go +++ b/store-client/pkg/client/mongodb_client.go @@ -299,7 +299,7 @@ func BuildQuarantineUpdatePipeline() any { return mongo.Pipeline{ bson.D{{Key: opMatch, Value: bson.D{ {Key: fieldOperationType, Value: "update"}, - {Key: "$or", Value: bson.A{ + {Key: opOr, Value: bson.A{ bson.D{{Key: fieldUpdatedFields, Value: bson.D{{Key: nodeQuarantinedStatusField, Value: model.Quarantined}}}}, bson.D{{Key: fieldUpdatedFields, @@ -834,6 +834,8 @@ func (c *MongoDBClient) CountDocuments(ctx context.Context, filter any, opts *Co // Aggregate performs an aggregation query func (c *MongoDBClient) Aggregate(ctx context.Context, pipeline any) (Cursor, error) { + pipeline, _ = ResolvePipelineOptions(pipeline) + // Convert datastore.Pipeline to mongo.Pipeline if needed var mongoPipeline any @@ -885,6 +887,8 @@ func (c *MongoDBClient) Ping(ctx context.Context) error { // NewChangeStreamWatcher creates a new change stream watcher using the existing implementation func (c *MongoDBClient) NewChangeStreamWatcher(ctx context.Context, tokenConfig TokenConfig, pipeline any) (ChangeStreamWatcher, error) { + pipeline, _ = ResolvePipelineOptions(pipeline) + // Convert to the existing configuration format mongoConfig := mongoWatcher.MongoDBConfig{ URI: c.config.GetConnectionURI(), diff --git a/store-client/pkg/client/mongodb_pipeline_builder.go b/store-client/pkg/client/mongodb_pipeline_builder.go index dabae56ba..7e7385263 100644 --- a/store-client/pkg/client/mongodb_pipeline_builder.go +++ b/store-client/pkg/client/mongodb_pipeline_builder.go @@ -142,6 +142,39 @@ func (b *MongoDBPipelineBuilder) BuildProcessableNonFatalUnhealthyInsertsPipelin ) } +// BuildAnalyzerHealthEventInsertsPipeline creates the analyzer input pipeline. +// It includes processable healthy events so configured recovery mappings can +// clear derived conditions, while still excluding analyzer output and +// observability-only STORE_ONLY events. +func (b *MongoDBPipelineBuilder) BuildAnalyzerHealthEventInsertsPipeline() datastore.Pipeline { + return datastore.ToPipeline( + datastore.D( + datastore.E(opMatch, datastore.D( + datastore.E(fieldOperationType, opTypeInsert), + datastore.E("fullDocument.healthevent.agent", datastore.D(datastore.E(opNE, "health-events-analyzer"))), + datastore.E("$or", datastore.A( + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + int32(protos.ProcessingStrategy_UNSPECIFIED), + )), + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), + )), + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + int32(protos.ProcessingStrategy_STORE_AND_ANALYSE), + )), + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + datastore.D(datastore.E("$exists", false)), + )), + )), + )), + ), + ) +} + // BuildQuarantinedAndDrainedNodesPipeline creates a pipeline for remediation-ready nodes // This watches for insert/update events where both quarantine and eviction status indicate the // node is ready for reboot, or where the node has been unquarantined and needs cleanup, or where diff --git a/store-client/pkg/client/permanent_error.go b/store-client/pkg/client/permanent_error.go new file mode 100644 index 000000000..2be329348 --- /dev/null +++ b/store-client/pkg/client/permanent_error.go @@ -0,0 +1,68 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +// permanentError marks a deterministic event failure that replay cannot fix. +type permanentError struct { + cause error +} + +func (e *permanentError) Error() string { return e.cause.Error() } +func (e *permanentError) Unwrap() error { return e.cause } +func (e *permanentError) permanent() {} + +// PermanentError marks an event-processing error as deterministic. The event +// processor checkpoints such events so one poison record cannot block later +// work indefinitely. +func PermanentError(err error) error { + if err == nil || IsPermanentError(err) { + return err + } + + return &permanentError{cause: err} +} + +// IsPermanentError reports whether the complete error represents a permanent +// failure. Joined errors are permanent only when every constituent is marked. +func IsPermanentError(err error) bool { + if err == nil { + return false + } + + if _, ok := err.(interface{ permanent() }); ok { + return true + } + + if joined, ok := err.(interface{ Unwrap() []error }); ok { + causes := joined.Unwrap() + if len(causes) == 0 { + return false + } + + for _, cause := range causes { + if !IsPermanentError(cause) { + return false + } + } + + return true + } + + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + return IsPermanentError(wrapped.Unwrap()) + } + + return false +} diff --git a/store-client/pkg/client/permanent_error_test.go b/store-client/pkg/client/permanent_error_test.go new file mode 100644 index 000000000..e038ff339 --- /dev/null +++ b/store-client/pkg/client/permanent_error_test.go @@ -0,0 +1,111 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + datamodels "github.com/nvidia/nvsentinel/data-models/pkg/model" + protos "github.com/nvidia/nvsentinel/data-models/pkg/protos" +) + +func TestPermanentError_WrappedAndJoinedErrors_ClassifiesOnlyFullyPermanentChains(t *testing.T) { + permanent := PermanentError(errors.New("invalid event")) + require.True(t, IsPermanentError(permanent)) + require.True(t, IsPermanentError(fmt.Errorf("wrapped: %w", permanent))) + require.True(t, IsPermanentError(errors.Join(permanent, PermanentError(errors.New("bad rule"))))) + require.False(t, IsPermanentError(errors.Join(permanent, errors.New("database unavailable")))) + require.False(t, IsPermanentError(nil)) +} + +func TestEventProcessor_PermanentHandlerFailure_CheckpointsAndContinues(t *testing.T) { + events := make(chan Event, 2) + events <- newPermanentErrorTestEvent("1") + events <- newPermanentErrorTestEvent("2") + close(events) + + watcher := &permanentErrorTestWatcher{events: events} + processor := NewEventProcessor(watcher, nil, EventProcessorConfig{}).(*DefaultEventProcessor) + var handled []string + processor.SetEventHandler(EventHandlerFunc(func(_ context.Context, event *datamodels.HealthEventWithStatus) error { + id := event.HealthEvent.GetId() + handled = append(handled, id) + if id == "1" { + return PermanentError(errors.New("deterministic failure")) + } + + return nil + })) + + require.NoError(t, processor.processEvents(context.Background())) + require.Equal(t, []string{"1", "2"}, handled) + require.Equal(t, []string{"1", "2"}, watcher.marked) +} + +func TestEventProcessor_PermanentFailureCheckpointFails_StopsAtUncheckpointedEvent(t *testing.T) { + events := make(chan Event, 1) + events <- newPermanentErrorTestEvent("1") + close(events) + + checkpointErr := errors.New("checkpoint unavailable") + watcher := &permanentErrorTestWatcher{events: events, markErr: checkpointErr} + processor := NewEventProcessor(watcher, nil, EventProcessorConfig{}).(*DefaultEventProcessor) + processor.SetEventHandler(EventHandlerFunc(func(context.Context, *datamodels.HealthEventWithStatus) error { + return PermanentError(errors.New("deterministic failure")) + })) + + err := processor.processEvents(context.Background()) + require.ErrorContains(t, err, "stopping at uncheckpointed event") + require.ErrorIs(t, err, checkpointErr) + require.Equal(t, []string{"1"}, watcher.marked) +} + +type permanentErrorTestEvent struct { + id string +} + +func newPermanentErrorTestEvent(id string) *permanentErrorTestEvent { + return &permanentErrorTestEvent{id: id} +} +func (e *permanentErrorTestEvent) GetDocumentID() (string, error) { return e.id, nil } +func (e *permanentErrorTestEvent) GetRecordUUID() (string, error) { return e.id, nil } +func (e *permanentErrorTestEvent) GetNodeName() (string, error) { return "node-a", nil } +func (e *permanentErrorTestEvent) GetResumeToken() []byte { return []byte(e.id) } +func (e *permanentErrorTestEvent) UnmarshalDocument(value any) error { + event := value.(*datamodels.HealthEventWithStatus) + event.HealthEvent = &protos.HealthEvent{Id: e.id} + + return nil +} + +type permanentErrorTestWatcher struct { + events chan Event + marked []string + markErr error +} + +func (w *permanentErrorTestWatcher) Start(context.Context) {} +func (w *permanentErrorTestWatcher) Events() <-chan Event { return w.events } +func (w *permanentErrorTestWatcher) Close(context.Context) error { return nil } +func (w *permanentErrorTestWatcher) MarkProcessed(_ context.Context, token []byte) error { + w.marked = append(w.marked, string(token)) + + return w.markErr +} diff --git a/store-client/pkg/client/pipeline_builder.go b/store-client/pkg/client/pipeline_builder.go index 4940b9dca..2dc967281 100644 --- a/store-client/pkg/client/pipeline_builder.go +++ b/store-client/pkg/client/pipeline_builder.go @@ -41,6 +41,11 @@ type PipelineBuilder interface { // Used by: health-events-analyzer to analyze EXECUTE_REMEDIATION and STORE_AND_ANALYSE source events. BuildProcessableNonFatalUnhealthyInsertsPipeline() datastore.Pipeline + // BuildAnalyzerHealthEventInsertsPipeline watches processable healthy and + // unhealthy source events. Health-events-analyzer uses healthy events for + // configured derived-condition recovery mappings. + BuildAnalyzerHealthEventInsertsPipeline() datastore.Pipeline + // BuildQuarantinedAndDrainedNodesPipeline creates a pipeline for remediation-ready nodes // Used by: fault-remediation to detect when nodes are ready for reboot BuildQuarantinedAndDrainedNodesPipeline() datastore.Pipeline diff --git a/store-client/pkg/client/pipeline_builder_test.go b/store-client/pkg/client/pipeline_builder_test.go index 355c3596a..1cae3034a 100644 --- a/store-client/pkg/client/pipeline_builder_test.go +++ b/store-client/pkg/client/pipeline_builder_test.go @@ -16,6 +16,7 @@ package client import ( "fmt" + "strings" "testing" "github.com/nvidia/nvsentinel/data-models/pkg/protos" @@ -110,6 +111,32 @@ func TestProcessableNonFatalUnhealthyInsertsPipeline(t *testing.T) { } } +func TestAnalyzerHealthEventInsertsPipeline(t *testing.T) { + testCases := []struct { + name string + builder PipelineBuilder + }{ + {"MongoDB", NewMongoDBPipelineBuilder()}, + {"PostgreSQL", NewPostgreSQLPipelineBuilder()}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + pipeline := tc.builder.BuildAnalyzerHealthEventInsertsPipeline() + require.Len(t, pipeline, 1) + + serialized := strings.ToLower(fmt.Sprint(pipeline)) + assert.NotContains(t, serialized, "ishealthy", + "analyzer input must admit healthy recovery events") + assert.Contains(t, serialized, "health-events-analyzer", + "analyzer output must remain excluded") + assert.Contains(t, serialized, "processingstrategy") + assert.Contains(t, serialized, fmt.Sprint(int32(protos.ProcessingStrategy_UNSPECIFIED))) + assert.Contains(t, serialized, fmt.Sprint(int32(protos.ProcessingStrategy_STORE_AND_ANALYSE))) + }) + } +} + func TestQuarantinedAndDrainedNodesPipeline(t *testing.T) { testCases := []struct { name string diff --git a/store-client/pkg/client/pipeline_options.go b/store-client/pkg/client/pipeline_options.go new file mode 100644 index 000000000..7bbe0491b --- /dev/null +++ b/store-client/pkg/client/pipeline_options.go @@ -0,0 +1,64 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +// PipelineOptions carries opt-in behavior without changing the pipeline seen +// by providers that do not need it. +type PipelineOptions struct { + Pipeline any + EnableExtendedFilters bool + ExtendedFilterPrefix int +} + +// WithExtendedFilters enables MongoDB-compatible logical and field-presence +// filters in PostgreSQL. Existing consumers retain their current semantics +// unless they opt in explicitly. +func WithExtendedFilters(pipeline any) PipelineOptions { + return PipelineOptions{Pipeline: pipeline, EnableExtendedFilters: true} +} + +// WithExtendedFilterPrefix enables extended PostgreSQL translation only for +// the leading aggregation stages owned by the caller. Later configured stages +// retain legacy translation semantics. +func WithExtendedFilterPrefix(pipeline any, stages int) PipelineOptions { + return PipelineOptions{Pipeline: pipeline, ExtendedFilterPrefix: max(stages, 0)} +} + +// ResolvePipelineOptions returns the underlying pipeline and whether extended +// PostgreSQL filter translation was requested. +func ResolvePipelineOptions(pipeline any) (any, bool) { + pipeline, options := ResolvePipelineStageOptions(pipeline) + + return pipeline, options.EnableExtendedFilters +} + +// ResolvePipelineStageOptions returns the underlying pipeline and its +// stage-scoped PostgreSQL filter options. +func ResolvePipelineStageOptions(pipeline any) (any, PipelineOptions) { + options, ok := pipeline.(PipelineOptions) + if !ok { + return pipeline, PipelineOptions{} + } + + return options.Pipeline, options +} + +func (o PipelineOptions) extendedFiltersForStage(stage int) bool { + return o.EnableExtendedFilters || stage < o.ExtendedFilterPrefix +} + +func (o PipelineOptions) rejectExtendedOperatorsForStage(stage int) bool { + return !o.EnableExtendedFilters && o.ExtendedFilterPrefix > 0 && stage >= o.ExtendedFilterPrefix +} diff --git a/store-client/pkg/client/pipeline_options_test.go b/store-client/pkg/client/pipeline_options_test.go new file mode 100644 index 000000000..5875e6450 --- /dev/null +++ b/store-client/pkg/client/pipeline_options_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package client + +import "testing" + +func TestResolvePipelineOptions(t *testing.T) { + pipeline := []any{map[string]any{"$match": map[string]any{"nodeName": "node-a"}}} + + raw, extended := ResolvePipelineOptions(WithExtendedFilters(pipeline)) + if !extended || len(raw.([]any)) != 1 { + t.Fatalf("extended options = %#v, %t", raw, extended) + } + + raw, extended = ResolvePipelineOptions(pipeline) + if extended || len(raw.([]any)) != 1 { + t.Fatalf("plain options = %#v, %t", raw, extended) + } + + _, options := ResolvePipelineStageOptions(WithExtendedFilterPrefix(pipeline, -1)) + if options.ExtendedFilterPrefix != 0 { + t.Fatalf("negative prefix = %d, want 0", options.ExtendedFilterPrefix) + } +} diff --git a/store-client/pkg/client/postgresql_client.go b/store-client/pkg/client/postgresql_client.go index a55004c2c..67c655070 100644 --- a/store-client/pkg/client/postgresql_client.go +++ b/store-client/pkg/client/postgresql_client.go @@ -18,8 +18,10 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "log/slog" + "reflect" "regexp" "sort" "strconv" @@ -39,13 +41,16 @@ const ( jsonbDocumentColumn = "document" // MongoDB query operators - opLTE = "$lte" - opEQ = "$eq" - opGTE = "$gte" - opGT = "$gt" - opLT = "$lt" - opNE = "$ne" - opIn = "$in" + opLTE = "$lte" + opEQ = "$eq" + opGTE = "$gte" + opGT = "$gt" + opLT = "$lt" + opNE = "$ne" + opIn = "$in" + opExists = "$exists" + opAnd = "$and" + opOr = "$or" // MongoDB aggregation stages and update operators opMatch = "$match" @@ -62,6 +67,9 @@ const ( // SQL constants orderDESC = "DESC" sqlTrueClause = "TRUE" + sqlDistinct = "IS DISTINCT FROM" + createdAtSQL = "created_at" + updatedAtSQL = "updated_at" // SQL window frame bounds frameBoundUnbounded = "ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING" @@ -634,6 +642,8 @@ func (c *PostgreSQLClient) CountDocuments(ctx context.Context, filter any, opts // //nolint:gocyclo,cyclop,gocognit // Complexity 11: handles pipeline type conversion and stage routing - acceptable func (c *PostgreSQLClient) Aggregate(ctx context.Context, pipeline any) (Cursor, error) { + pipeline, pipelineOptions := ResolvePipelineStageOptions(pipeline) + // Convert pipeline to slice of stages var stages []map[string]any @@ -672,7 +682,7 @@ func (c *PostgreSQLClient) Aggregate(ctx context.Context, pipeline any) (Cursor, } // Build SQL query from pipeline stages - query, args, err := c.buildAggregationQuery(stages) + query, args, err := c.buildAggregationQuery(stages, pipelineOptions) if err != nil { return nil, err } @@ -766,6 +776,13 @@ func resolveSQLFilter(filter any) (string, []any, bool) { // buildWhereClause converts filters to a PostgreSQL WHERE clause. // Accepts a *query.Builder (via ToSQL interface) or a legacy map[string]interface{} filter. func (c *PostgreSQLClient) buildWhereClause(filter any) (string, []any, error) { + return c.buildWhereClauseWithOptions(filter, false) +} + +func (c *PostgreSQLClient) buildWhereClauseWithOptions( + filter any, + extendedFilters bool, +) (string, []any, error) { if filter == nil { return sqlTrueClause, []any{}, nil } @@ -787,17 +804,38 @@ func (c *PostgreSQLClient) buildWhereClause(filter any) (string, []any, error) { return sqlTrueClause, []any{}, nil } + return c.buildWhereClauseMapWithOptions(filterMap, 1, extendedFilters) +} + +//nolint:cyclop // Handles the small set of opt-in logical and field filter forms. +func (c *PostgreSQLClient) buildWhereClauseMapWithOptions( + filterMap map[string]any, + startParam int, + extendedFilters bool, +) (string, []any, error) { + if len(filterMap) == 0 { + return sqlTrueClause, nil, nil + } + var ( conditions []string args []any ) - paramCount := 1 + paramCount := startParam + keys := make([]string, 0, len(filterMap)) + + for key := range filterMap { + keys = append(keys, key) + } + + sort.Strings(keys) + + for _, key := range keys { + value := filterMap[key] - for key, value := range filterMap { - // Handle special operators - if key == "$expr" { - // Handle $expr operator which allows aggregation expressions in match + switch key { + case "$expr": exprCondition, err := c.buildExprCondition(value) if err != nil { return "", nil, err @@ -805,15 +843,28 @@ func (c *PostgreSQLClient) buildWhereClause(filter any) (string, []any, error) { conditions = append(conditions, exprCondition) + continue + case opAnd, opOr: + if !extendedFilters { + break + } + + condition, logicalArgs, err := c.buildLogicalWhereClause(key, value, paramCount, extendedFilters) + if err != nil { + return "", nil, err + } + + conditions = append(conditions, condition) + args = append(args, logicalArgs...) + paramCount += len(logicalArgs) + continue } // Check if value is a map containing comparison operators if valueMap, ok := value.(map[string]any); ok { // Handle comparison operators: {"count": {opGTE: 5}} → document->>'count' >= 5 - jsonPath := c.buildJSONPath(key) - - condition, valueArgs, err := c.buildFieldComparison(jsonPath, valueMap, paramCount) + condition, valueArgs, err := c.buildFieldComparison(key, valueMap, paramCount, extendedFilters) if err != nil { return "", nil, err } @@ -828,10 +879,11 @@ func (c *PostgreSQLClient) buildWhereClause(filter any) (string, []any, error) { // Simple equality check on JSONB fields // Example: {"nodeName": "node-1"} → document->>'nodeName' = $1 // Example: {"healthevent.nodename": "node-1"} → document->'healthevent'->>'nodename' = $1 - jsonPath := c.buildJSONPath(key) - conditions = append(conditions, fmt.Sprintf("%s = $%d", jsonPath, paramCount)) - args = append(args, value) - paramCount++ + jsonPath := c.buildJSONPathWithOptions(key, extendedFilters) + condition, valueArgs := buildScalarComparison(jsonPath, "=", value, paramCount, extendedFilters) + conditions = append(conditions, condition) + args = append(args, valueArgs...) + paramCount += len(valueArgs) } whereClause := strings.Join(conditions, " AND ") @@ -839,37 +891,116 @@ func (c *PostgreSQLClient) buildWhereClause(filter any) (string, []any, error) { return whereClause, args, nil } +func (c *PostgreSQLClient) buildLogicalWhereClause( + operator string, + value any, + startParam int, + extendedFilters bool, +) (string, []any, error) { + logicalValues, ok := logicalFilterValues(value) + if !ok || len(logicalValues) == 0 { + return "", nil, datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("%s must contain at least one filter", operator), + fmt.Errorf("got type %T", value), + ) + } + + logicalConditions := make([]string, 0, len(logicalValues)) + logicalArgs := make([]any, 0) + paramCount := startParam + + for i, logicalValue := range logicalValues { + logicalMap, ok := logicalValue.(map[string]any) + if !ok { + return "", nil, datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("%s filter %d must be a map", operator, i), + fmt.Errorf("got type %T", logicalValue), + ) + } + + condition, args, err := c.buildWhereClauseMapWithOptions(logicalMap, paramCount, extendedFilters) + if err != nil { + return "", nil, fmt.Errorf("build %s filter %d: %w", operator, i, err) + } + + logicalConditions = append(logicalConditions, condition) + logicalArgs = append(logicalArgs, args...) + paramCount += len(args) + } + + joiner := " OR " + if operator == opAnd { + joiner = " AND " + } + + return "(" + strings.Join(logicalConditions, joiner) + ")", logicalArgs, nil +} + +func logicalFilterValues(value any) ([]any, bool) { + switch value := value.(type) { + case []any: + return value, true + case []map[string]any: + values := make([]any, len(value)) + for i := range value { + values[i] = value[i] + } + + return values, true + case datastore.Array: + return []any(value), true + default: + return nil, false + } +} + // buildFieldComparison builds a comparison condition for a field with operators // Handles operators like $gte, $gt, $lte, $lt, $eq, $ne func (c *PostgreSQLClient) buildFieldComparison( - jsonPath string, + fieldPath string, operators map[string]any, startParam int, + extendedFilters bool, ) (string, []any, error) { var ( conditions []string args []any ) + jsonPath := c.buildJSONPathWithOptions(fieldPath, extendedFilters) paramCount := startParam - for op, value := range operators { - var sqlOp string + operatorNames := make([]string, 0, len(operators)) - switch op { - case opGTE: - sqlOp = ">=" - case opGT: - sqlOp = ">" - case opLTE: - sqlOp = "<=" - case opLT: - sqlOp = "<" - case opEQ: - sqlOp = "=" - case opNE: - sqlOp = "!=" - default: + for op := range operators { + operatorNames = append(operatorNames, op) + } + + sort.Strings(operatorNames) + + for _, op := range operatorNames { + value := operators[op] + + if op == opExists && extendedFilters { + existsPath := jsonPath + if jsonPath != createdAtSQL && jsonPath != updatedAtSQL { + existsPath = c.buildJSONPathAsJSONBWithOptions(fieldPath, true) + } + + condition, err := buildExistsCondition(existsPath, value) + if err != nil { + return "", nil, err + } + + conditions = append(conditions, condition) + + continue + } + + sqlOperator, ok := sqlComparisonOperator(op) + if !ok { return "", nil, datastore.NewQueryError( datastore.ProviderPostgreSQL, fmt.Sprintf("unsupported comparison operator: %s", op), @@ -877,9 +1008,10 @@ func (c *PostgreSQLClient) buildFieldComparison( ) } - conditions = append(conditions, fmt.Sprintf("%s %s $%d", jsonPath, sqlOp, paramCount)) - args = append(args, value) - paramCount++ + condition, valueArgs := buildScalarComparison(jsonPath, sqlOperator, value, paramCount, extendedFilters) + conditions = append(conditions, condition) + args = append(args, valueArgs...) + paramCount += len(valueArgs) } condition := strings.Join(conditions, " AND ") @@ -887,11 +1019,144 @@ func (c *PostgreSQLClient) buildFieldComparison( return condition, args, nil } +func buildExistsCondition(jsonPath string, value any) (string, error) { + exists, ok := value.(bool) + if !ok { + return "", datastore.NewValidationError( + datastore.ProviderPostgreSQL, + "$exists value must be a boolean", + fmt.Errorf("got type %T", value), + ) + } + + if exists { + return jsonPath + " IS NOT NULL", nil + } + + return jsonPath + " IS NULL", nil +} + +func sqlComparisonOperator(operator string) (string, bool) { + switch operator { + case opGTE: + return ">=", true + case opGT: + return ">", true + case opLTE: + return "<=", true + case opLT: + return "<", true + case opEQ: + return "=", true + case opNE: + return sqlDistinct, true + default: + return "", false + } +} + +func buildScalarComparison( + jsonPath, operator string, + value any, + parameter int, + typed bool, +) (string, []any) { + if !typed { + return fmt.Sprintf("%s %s $%d", jsonPath, operator, parameter), []any{value} + } + + if value == nil { + if operator == "!=" || operator == sqlDistinct { + return jsonPath + " IS NOT NULL", nil + } + + return jsonPath + " IS NULL", nil + } + + expression, argument := typedComparisonOperand(jsonPath, value) + if operator == sqlDistinct { + reflected := reflect.ValueOf(value) + if reflected.IsValid() && reflected.Kind() == reflect.Bool { + expression = fmt.Sprintf( + "CASE WHEN %s IN ('true', 'false') THEN (%s)::boolean END", + jsonPath, jsonPath, + ) + } + } + + return fmt.Sprintf("%s %s $%d", expression, operator, parameter), []any{argument} +} + +func typedComparisonOperand(jsonPath string, value any) (string, any) { + if jsonPath == createdAtSQL || jsonPath == updatedAtSQL { + return jsonPath, value + } + + reflected := reflect.ValueOf(value) + if !reflected.IsValid() { + return jsonPath, value + } + + if reflected.Kind() == reflect.Bool { + return fmt.Sprintf( + "CASE WHEN %s IS NULL THEN false WHEN %s IN ('true', 'false') THEN (%s)::boolean END", + jsonPath, jsonPath, jsonPath, + ), value + } + + if isJSONNumber(reflected.Kind()) { + return fmt.Sprintf( + "CASE WHEN %s ~ '^-?[0-9]+([.][0-9]+)?([eE][+-]?[0-9]+)?$' THEN (%s)::numeric END", + jsonPath, jsonPath, + ), formatJSONNumber(reflected, value) + } + + return jsonPath, value +} + +func isJSONNumber(kind reflect.Kind) bool { + return kind >= reflect.Int && kind <= reflect.Float64 && kind != reflect.Uintptr +} + +func formatJSONNumber(reflected reflect.Value, fallback any) any { + kind := reflected.Kind() + if kind >= reflect.Int && kind <= reflect.Int64 { + return strconv.FormatInt(reflected.Int(), 10) + } + + if kind >= reflect.Uint && kind <= reflect.Uint64 { + return strconv.FormatUint(reflected.Uint(), 10) + } + + if kind == reflect.Float32 || kind == reflect.Float64 { + return strconv.FormatFloat(reflected.Float(), 'g', -1, reflected.Type().Bits()) + } + + return fallback +} + // buildExprCondition converts MongoDB $expr operator to PostgreSQL SQL // This handles aggregation expressions used in $match stages -// -//nolint:cyclop,gocognit // Complexity acceptable: handles logical and comparison operators in $match $expr func (c *PostgreSQLClient) buildExprCondition(expr any) (string, error) { + condition, err := c.buildExprConditionValue(expr) + if err == nil { + return condition, nil + } + + var datastoreErr *datastore.DatastoreError + if errors.As(err, &datastoreErr) { + return "", err + } + + return "", datastore.NewValidationError( + datastore.ProviderPostgreSQL, + "invalid $expr", + err, + ) +} + +//nolint:cyclop,gocognit // Complexity acceptable: handles logical and comparison operators in $match $expr. +func (c *PostgreSQLClient) buildExprConditionValue(expr any) (string, error) { exprMap, ok := expr.(map[string]any) if !ok { return "", datastore.NewValidationError( @@ -901,10 +1166,18 @@ func (c *PostgreSQLClient) buildExprCondition(expr any) (string, error) { ) } + if len(exprMap) != 1 { + return "", datastore.NewValidationError( + datastore.ProviderPostgreSQL, + "$expr must contain exactly one operator", + fmt.Errorf("got %d operators", len(exprMap)), + ) + } + // Handle logical and comparison operators for op, value := range exprMap { switch op { - case "$and": + case opAnd: // Handle logical AND andArray, ok := value.([]any) if !ok { @@ -936,7 +1209,7 @@ func (c *PostgreSQLClient) buildExprCondition(expr any) (string, error) { return fmt.Sprintf("(%s)", strings.Join(conditions, " AND ")), nil - case "$or": + case opOr: // Handle logical OR orArray, ok := value.([]any) if !ok { @@ -1368,7 +1641,7 @@ func (c *PostgreSQLClient) buildExprValue(value any) (string, error) { "sql", sql) return sql, nil - case "$and": + case opAnd: // $and performs logical AND on array of expressions: [expr1, expr2, ...] // PostgreSQL: (expr1 AND expr2 AND ...) operandArray, ok := operand.([]any) @@ -1716,6 +1989,14 @@ func normalizeFieldName(fieldName string) string { return fieldName } +func normalizeFieldNameWithOptions(fieldName string, extendedFilters bool) string { + if extendedFilters && fieldName == "processingstrategy" { + return "processingStrategy" + } + + return normalizeFieldName(fieldName) +} + // buildJSONPathAsJSONB converts a MongoDB-style field path to PostgreSQL JSONB path expression // that preserves JSONB type (using -> for all parts, including the last one). // This is used in aggregation expressions where we need to preserve arrays/objects. @@ -1725,12 +2006,19 @@ func normalizeFieldName(fieldName string) string { // "healthevent.entitiesimpacted" → "document->'healthevent'->'entitiesImpacted'" // "status.metadata" → "document->'status'->'metadata'" func (c *PostgreSQLClient) buildJSONPathAsJSONB(fieldPath string) string { + return c.buildJSONPathAsJSONBWithOptions(fieldPath, false) +} + +func (c *PostgreSQLClient) buildJSONPathAsJSONBWithOptions( + fieldPath string, + extendedFilters bool, +) string { parts := strings.Split(fieldPath, ".") path := jsonbDocumentColumn for _, part := range parts { // Normalize field name to camelCase and use -> to keep JSONB type - normalizedPart := normalizeFieldName(part) + normalizedPart := normalizeFieldNameWithOptions(part, extendedFilters) path = fmt.Sprintf("%s->'%s'", path, normalizedPart) } @@ -1745,11 +2033,24 @@ func (c *PostgreSQLClient) buildJSONPathAsJSONB(fieldPath string) string { // "healthevent.nodename" → "document->'healthevent'->>'nodeName'" // "status.message" → "document->'status'->>'message'" func (c *PostgreSQLClient) buildJSONPath(fieldPath string) string { + return c.buildJSONPathWithOptions(fieldPath, false) +} + +func (c *PostgreSQLClient) buildJSONPathWithOptions(fieldPath string, extendedFilters bool) string { + if extendedFilters { + switch strings.ToLower(fieldPath) { + case "createdat": + return createdAtSQL + case "updatedat": + return updatedAtSQL + } + } + parts := strings.Split(fieldPath, ".") if len(parts) == 1 { // Simple field: document->>'fieldName' - normalizedPart := normalizeFieldName(parts[0]) + normalizedPart := normalizeFieldNameWithOptions(parts[0], extendedFilters) return fmt.Sprintf("%s->>'%s'", jsonbDocumentColumn, normalizedPart) } @@ -1760,7 +2061,7 @@ func (c *PostgreSQLClient) buildJSONPath(fieldPath string) string { path := jsonbDocumentColumn for i, part := range parts { - normalizedPart := normalizeFieldName(part) + normalizedPart := normalizeFieldNameWithOptions(part, extendedFilters) if i == len(parts)-1 { // Last part: use ->> to get text value path = fmt.Sprintf("%s->>'%s'", path, normalizedPart) @@ -1828,19 +2129,30 @@ func (c *PostgreSQLClient) convertDatastoreValue(value any) any { } // buildAggregationQuery builds SQL query from MongoDB aggregation pipeline stages -func (c *PostgreSQLClient) buildAggregationQuery(stages []map[string]any) (string, []any, error) { +func (c *PostgreSQLClient) buildAggregationQuery( + stages []map[string]any, + options PipelineOptions, +) (string, []any, error) { builder := &aggregationQueryBuilder{ client: c, query: fmt.Sprintf("SELECT id, document FROM %s", c.table), } for i, stage := range stages { + builder.extendedFilters = options.extendedFiltersForStage(i) + builder.rejectExtendedOperators = options.rejectExtendedOperatorsForStage(i) + if err := builder.processStage(i, stage); err != nil { return "", nil, err } } - return builder.buildFinalQuery(), builder.args, nil + query, err := builder.buildFinalQuery() + if err != nil { + return "", nil, err + } + + return query, builder.args, nil } // aggregationQueryBuilder helps build aggregation queries with reduced complexity @@ -1859,7 +2171,9 @@ type aggregationQueryBuilder struct { addFields map[string]any // Fields to add via $addFields // postCountMatch stores $match conditions that come AFTER $count // These filter the count result, not the source rows - postCountMatch map[string]any + postCountMatch map[string]any + extendedFilters bool + rejectExtendedOperators bool } // windowFieldsSpec holds the specification for $setWindowFields @@ -1904,7 +2218,7 @@ func (b *aggregationQueryBuilder) processStage(stageIndex int, stage map[string] case "$addFields": return b.processAddFields(value) case "$project", "$lookup", "$unwind", "$facet": - return datastore.NewQueryError( + return datastore.NewValidationError( datastore.ProviderPostgreSQL, fmt.Sprintf("aggregation operator %s not yet supported", operator), fmt.Errorf("complex aggregation requires custom SQL implementation"), @@ -1931,6 +2245,12 @@ func (b *aggregationQueryBuilder) processMatch(value any) error { ) } + if b.extendedFilters || b.rejectExtendedOperators { + if err := validateAnalyzerMatchShape(matchMap, b.extendedFilters); err != nil { + return err + } + } + // If $count has already been processed, this is a post-count $match // that should filter the count result, not the source rows if b.isCount { @@ -1939,7 +2259,7 @@ func (b *aggregationQueryBuilder) processMatch(value any) error { return nil } - whereClause, matchArgs, err := b.client.buildWhereClause(matchMap) + whereClause, matchArgs, err := b.client.buildWhereClauseWithOptions(matchMap, b.extendedFilters) if err != nil { return err } @@ -1951,6 +2271,100 @@ func (b *aggregationQueryBuilder) processMatch(value any) error { return nil } +func validateAnalyzerMatchShape(matchMap map[string]any, extendedFilters bool) error { + for field, value := range matchMap { + if err := validateAnalyzerMatchEntry(field, value, extendedFilters); err != nil { + return err + } + } + + return nil +} + +func validateAnalyzerMatchEntry(field string, value any, extendedFilters bool) error { + if field == "$expr" { + return nil + } + + if field == opAnd || field == opOr { + return validateAnalyzerLogicalMatch(field, value, extendedFilters) + } + + if strings.HasPrefix(field, "$") { + return datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("aggregation operator %s is not supported in PostgreSQL $match", field), + nil, + ) + } + + return validateAnalyzerFieldMatch(field, value) +} + +func validateAnalyzerLogicalMatch(operator string, value any, extendedFilters bool) error { + if !extendedFilters { + return datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("aggregation operator %s requires extended filters", operator), + nil, + ) + } + + logicalValues, ok := logicalFilterValues(value) + if !ok || len(logicalValues) == 0 { + return nil + } + + for _, logicalValue := range logicalValues { + logicalMap, ok := logicalValue.(map[string]any) + if !ok { + continue + } + + if err := validateAnalyzerMatchShape(logicalMap, true); err != nil { + return err + } + } + + return nil +} + +func validateAnalyzerFieldMatch(field string, value any) error { + if operators, ok := value.(map[string]any); ok { + for operator, operand := range operators { + if _, supported := sqlComparisonOperator(operator); supported && isCompositeMatchValue(operand) { + return nonScalarMatchValueError(field, operand) + } + } + + return nil + } + + if isCompositeMatchValue(value) { + return nonScalarMatchValueError(field, value) + } + + return nil +} + +func isCompositeMatchValue(value any) bool { + if value == nil { + return false + } + + kind := reflect.ValueOf(value).Kind() + + return kind == reflect.Array || kind == reflect.Map || kind == reflect.Slice +} + +func nonScalarMatchValueError(field string, value any) error { + return datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("$match field %q requires a scalar comparison value", field), + fmt.Errorf("got type %T", value), + ) +} + func (b *aggregationQueryBuilder) processSort(value any) error { sortMap, ok := value.(map[string]any) if !ok { @@ -2171,7 +2585,7 @@ func (b *aggregationQueryBuilder) processAddFields(value any) error { return nil } -func (b *aggregationQueryBuilder) buildFinalQuery() string { +func (b *aggregationQueryBuilder) buildFinalQuery() (string, error) { // Handle $count operator if b.isCount { return b.buildCountQuery() @@ -2179,25 +2593,25 @@ func (b *aggregationQueryBuilder) buildFinalQuery() string { // Handle $group operator if b.groupBy != nil { - return b.buildGroupQuery() + return b.buildGroupQuery(), nil } // Handle $setWindowFields operator if b.windowFields != nil { - return b.buildWindowFieldsQuery() + return b.buildWindowFieldsQuery(), nil } // Handle $addFields operator if b.addFields != nil { - return b.buildAddFieldsQuery() + return b.buildAddFieldsQuery(), nil } // Standard query - return b.buildStandardQuery() + return b.buildStandardQuery(), nil } // buildCountQuery builds the SQL for $count aggregation with optional post-count filtering -func (b *aggregationQueryBuilder) buildCountQuery() string { +func (b *aggregationQueryBuilder) buildCountQuery() (string, error) { subquery := b.query if len(b.whereClauses) > 0 { subquery += " WHERE " + strings.Join(b.whereClauses, " AND ") @@ -2212,7 +2626,7 @@ func (b *aggregationQueryBuilder) buildCountQuery() string { return b.buildPostCountFilter(countQuery) } - return countQuery + return countQuery, nil } // buildStandardQuery builds a standard SELECT query with WHERE, ORDER BY, LIMIT, OFFSET @@ -2241,51 +2655,105 @@ func (b *aggregationQueryBuilder) buildStandardQuery() string { // buildPostCountFilter wraps a count query with a WHERE clause to filter the count result. // This handles the MongoDB pattern: $count -> $match (filter on count) // Example: {$match: {count: {$gte: 5}}} after $count should return empty if count < 5 -func (b *aggregationQueryBuilder) buildPostCountFilter(countQuery string) string { +func (b *aggregationQueryBuilder) buildPostCountFilter(countQuery string) (string, error) { // Build WHERE conditions for the count result conditions := []string{} + fields := make([]string, 0, len(b.postCountMatch)) - for field, value := range b.postCountMatch { - condition := b.buildPostCountCondition(field, value) - if condition != "" { - conditions = append(conditions, condition) + for field := range b.postCountMatch { + fields = append(fields, field) + } + + sort.Strings(fields) + + for _, field := range fields { + value := b.postCountMatch[field] + + condition, err := b.buildPostCountCondition(field, value) + if err != nil { + return "", err } + + conditions = append(conditions, condition) } if len(conditions) == 0 { - return countQuery + return "", datastore.NewValidationError( + datastore.ProviderPostgreSQL, + "post-count $match must contain at least one supported condition", + nil, + ) } // Wrap the count query and apply the filter on the result // The count result is in document->>'countField', so we filter on that return fmt.Sprintf("SELECT * FROM (%s) as count_result WHERE %s", - countQuery, strings.Join(conditions, " AND ")) + countQuery, strings.Join(conditions, " AND ")), nil } // buildPostCountCondition builds a single condition for filtering count results -func (b *aggregationQueryBuilder) buildPostCountCondition(field string, value any) string { +func (b *aggregationQueryBuilder) buildPostCountCondition(field string, value any) (string, error) { + if strings.HasPrefix(field, "$") { + return "", datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("operator %s is not supported in a post-count $match", field), + nil, + ) + } + // The count result is stored as document->>'field' // We need to cast it to a number for comparison fieldPath := fmt.Sprintf("(document->>'%s')::bigint", field) switch v := value.(type) { case map[string]any: - // Handle comparison operators like {$gte: 5} - for op, opValue := range v { - if sqlOp := b.mapComparisonOperator(op); sqlOp != "" { - b.args = append(b.args, opValue) + if len(v) == 0 { + return "", datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("post-count $match field %q must contain a comparison operator", field), + nil, + ) + } + + operatorNames := make([]string, 0, len(v)) + for op := range v { + operatorNames = append(operatorNames, op) + } + + sort.Strings(operatorNames) - return fmt.Sprintf("%s %s $%d", fieldPath, sqlOp, len(b.args)) + conditions := make([]string, 0, len(operatorNames)) + for _, op := range operatorNames { + opValue := v[op] + + sqlOp := b.mapComparisonOperator(op) + if sqlOp == "" { + return "", datastore.NewValidationError( + datastore.ProviderPostgreSQL, + fmt.Sprintf("unsupported post-count comparison operator: %s", op), + nil, + ) + } + + if isCompositeMatchValue(opValue) { + return "", nonScalarMatchValueError(field, opValue) } + + b.args = append(b.args, opValue) + conditions = append(conditions, fmt.Sprintf("%s %s $%d", fieldPath, sqlOp, len(b.args))) } + + return strings.Join(conditions, " AND "), nil default: + if isCompositeMatchValue(v) { + return "", nonScalarMatchValueError(field, v) + } + // Direct equality comparison b.args = append(b.args, v) - return fmt.Sprintf("%s = $%d", fieldPath, len(b.args)) + return fmt.Sprintf("%s = $%d", fieldPath, len(b.args)), nil } - - return "" } // mapComparisonOperator maps MongoDB comparison operators to SQL operators diff --git a/store-client/pkg/client/postgresql_client_test.go b/store-client/pkg/client/postgresql_client_test.go index cc6ed67bb..a6a816b65 100644 --- a/store-client/pkg/client/postgresql_client_test.go +++ b/store-client/pkg/client/postgresql_client_test.go @@ -16,13 +16,17 @@ package client import ( "context" + "reflect" "regexp" "strings" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/nvidia/nvsentinel/store-client/pkg/datastore" ) // TestPostgreSQLClient_BasicOperations tests basic CRUD operations @@ -146,6 +150,16 @@ func TestBuildJSONPath(t *testing.T) { field: "healthevent.status.message", expected: "document->'healthevent'->'status'->>'message'", }, + { + name: "createdAt legacy document field", + field: "createdAt", + expected: "document->>'createdAt'", + }, + { + name: "updatedAt legacy document field", + field: "updatedAt", + expected: "document->>'updatedAt'", + }, } for _, tt := range tests { @@ -156,6 +170,541 @@ func TestBuildJSONPath(t *testing.T) { } }) } + + if got := client.buildJSONPathWithOptions("createdAt", true); got != "created_at" { + t.Fatalf("extended createdAt path = %q", got) + } + if got := client.buildJSONPathWithOptions("updatedAt", true); got != "updated_at" { + t.Fatalf("extended updatedAt path = %q", got) + } +} + +func TestRecoveryQueryLogicalFilters(t *testing.T) { + client := &PostgreSQLClient{} + + for name, value := range map[string]any{ + "generic slice": []any{map[string]any{"nodeName": "node-a"}}, + "map slice": []map[string]any{{"nodeName": "node-a"}}, + "datastore array": datastore.Array{ + map[string]any{"nodeName": "node-a"}, + }, + } { + t.Run(name, func(t *testing.T) { + clause, args, err := client.buildLogicalWhereClause(opOr, value, 3, true) + if err != nil { + t.Fatalf("buildLogicalWhereClause() error = %v", err) + } + if clause != "(document->>'nodeName' = $3)" { + t.Fatalf("clause = %q", clause) + } + if !reflect.DeepEqual(args, []any{"node-a"}) { + t.Fatalf("args = %#v", args) + } + }) + } + + clause, args, err := client.buildLogicalWhereClause(opAnd, []any{ + map[string]any{"nodeName": "node-a"}, + map[string]any{"createdAt": map[string]any{opGT: time.Unix(10, 0)}}, + }, 1, true) + if err != nil { + t.Fatalf("buildLogicalWhereClause() error = %v", err) + } + if clause != "(document->>'nodeName' = $1 AND created_at > $2)" || len(args) != 2 { + t.Fatalf("clause = %q, args = %#v", clause, args) + } +} + +func TestRecoveryQueryLogicalFilterValidation(t *testing.T) { + client := &PostgreSQLClient{} + + for name, value := range map[string]any{ + "wrong type": "node-a", + "empty": []any{}, + "non-map": []any{"node-a"}, + "nested error": []any{ + map[string]any{"createdAt": map[string]any{"$unsupported": 1}}, + }, + } { + t.Run(name, func(t *testing.T) { + _, _, err := client.buildLogicalWhereClause(opOr, value, 1, true) + if err == nil { + t.Fatal("buildLogicalWhereClause() expected an error") + } + }) + } + + _, _, err := client.buildWhereClauseMapWithOptions(map[string]any{opOr: "node-a"}, 1, true) + if err == nil { + t.Fatal("root logical filter accepted an invalid value") + } + _, _, err = client.buildWhereClauseMapWithOptions(map[string]any{ + "status.value": map[string]any{opExists: "true"}, + }, 1, true) + if err == nil { + t.Fatal("field comparison accepted an invalid $exists value") + } +} + +func TestRecoveryQueryComparisonOperators(t *testing.T) { + for operator, expected := range map[string]string{ + opGTE: ">=", opGT: ">", opLTE: "<=", opLT: "<", opEQ: "=", opNE: "IS DISTINCT FROM", + } { + actual, ok := sqlComparisonOperator(operator) + if !ok || actual != expected { + t.Fatalf("sqlComparisonOperator(%q) = %q, %v", operator, actual, ok) + } + } + if _, ok := sqlComparisonOperator("$unsupported"); ok { + t.Fatal("unsupported operator accepted") + } + + for _, test := range []struct { + value any + wantExpression string + wantArgument any + }{ + {true, "CASE WHEN document->>'count' IS NULL THEN false WHEN document->>'count' IN ('true', 'false') THEN (document->>'count')::boolean END", true}, + {int32(-2), "CASE WHEN document->>'count' ~ '^-?[0-9]+([.][0-9]+)?([eE][+-]?[0-9]+)?$' THEN (document->>'count')::numeric END", "-2"}, + {uint64(3), "CASE WHEN document->>'count' ~ '^-?[0-9]+([.][0-9]+)?([eE][+-]?[0-9]+)?$' THEN (document->>'count')::numeric END", "3"}, + {float32(1.5), "CASE WHEN document->>'count' ~ '^-?[0-9]+([.][0-9]+)?([eE][+-]?[0-9]+)?$' THEN (document->>'count')::numeric END", "1.5"}, + } { + expression, argument := typedComparisonOperand("document->>'count'", test.value) + if expression != test.wantExpression || argument != test.wantArgument { + t.Fatalf("typedComparisonOperand(%#v) = %q, %#v", test.value, expression, argument) + } + } + cutoff := time.Unix(20, 0) + expression, argument := typedComparisonOperand("updated_at", cutoff) + if expression != "updated_at" || argument != cutoff { + t.Fatalf("updated_at operand = %q, %#v", expression, argument) + } + expression, argument = typedComparisonOperand("document->>'value'", "raw") + if expression != "document->>'value'" || argument != "raw" { + t.Fatalf("string operand = %q, %#v", expression, argument) + } + if condition, args := buildScalarComparison("document->>'value'", "=", nil, 1, true); condition != "document->>'value' IS NULL" || len(args) != 0 { + t.Fatalf("nil equality = %q, %#v", condition, args) + } + if condition, args := buildScalarComparison("document->>'value'", "!=", nil, 1, true); condition != "document->>'value' IS NOT NULL" || len(args) != 0 { + t.Fatalf("nil inequality = %q, %#v", condition, args) + } + condition, args := buildScalarComparison("document->>'value'", "IS DISTINCT FROM", "analyzer", 1, true) + assert.Equal(t, "document->>'value' IS DISTINCT FROM $1", condition) + assert.Equal(t, []any{"analyzer"}, args) + + condition, args = buildScalarComparison("document->>'value'", "IS DISTINCT FROM", false, 1, true) + assert.Equal(t, "CASE WHEN document->>'value' IN ('true', 'false') THEN (document->>'value')::boolean END IS DISTINCT FROM $1", condition) + assert.Equal(t, []any{false}, args) +} + +func TestRecoveryQueryExistsOperator(t *testing.T) { + path := "document->>'value'" + for value, expected := range map[bool]string{true: path + " IS NOT NULL", false: path + " IS NULL"} { + actual, err := buildExistsCondition(path, value) + if err != nil || actual != expected { + t.Fatalf("buildExistsCondition(%v) = %q, %v", value, actual, err) + } + } + if _, err := buildExistsCondition(path, "true"); err == nil { + t.Fatal("non-boolean $exists value accepted") + } + + client := &PostgreSQLClient{} + clause, args, err := client.buildWhereClauseWithOptions(map[string]any{ + "healthevent.processingstrategy": map[string]any{opExists: false}, + }, true) + if err != nil { + t.Fatal(err) + } + if clause != "document->'healthevent'->'processingStrategy' IS NULL" || len(args) != 0 { + t.Fatalf("$exists clause = %q, args = %#v", clause, args) + } +} + +func TestExtendedQueryTranslationRequiresOptIn(t *testing.T) { + client := &PostgreSQLClient{} + filter := map[string]any{opOr: []any{ + map[string]any{"healthevent.processingstrategy": int32(1)}, + map[string]any{"healthevent.processingstrategy": map[string]any{opExists: false}}, + }} + + legacyClause, _, err := client.buildWhereClause(filter) + if err != nil { + t.Fatal(err) + } + if strings.Contains(legacyClause, " OR ") || strings.Contains(legacyClause, "processingStrategy") { + t.Fatalf("unscoped filter enabled extended translation: %s", legacyClause) + } + + extendedClause, _, err := client.buildWhereClauseWithOptions(filter, true) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(extendedClause, " OR ") || !strings.Contains(extendedClause, "processingStrategy") { + t.Fatalf("scoped filter did not enable extended translation: %s", extendedClause) + } + + legacyNilClause, legacyNilArgs, err := client.buildWhereClause(map[string]any{"value": nil}) + if err != nil { + t.Fatal(err) + } + if legacyNilClause != "document->>'value' = $1" || len(legacyNilArgs) != 1 || legacyNilArgs[0] != nil { + t.Fatalf("unscoped nil comparison changed semantics: %s, %#v", legacyNilClause, legacyNilArgs) + } +} + +func TestAggregationExtendedFilterPrefix(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + stages := []map[string]any{ + {"$match": map[string]any{opOr: []any{ + map[string]any{"healthevent.processingstrategy": int32(0)}, + map[string]any{"healthevent.processingstrategy": int32(1)}, + map[string]any{"healthevent.processingstrategy": map[string]any{opExists: false}}, + }}}, + {"$match": map[string]any{"createdAt": "legacy-custom-rule"}}, + } + + rawPipeline, options := ResolvePipelineStageOptions(WithExtendedFilterPrefix(stages, 1)) + query, args, err := client.buildAggregationQuery(rawPipeline.([]map[string]any), options) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(query, "processingStrategy") || !strings.Contains(query, " OR ") { + t.Fatalf("mandatory stage did not use extended translation: %s", query) + } + if !strings.Contains(query, "document->>'createdAt' = $3") { + t.Fatalf("configured stage did not retain legacy translation: %s", query) + } + if len(args) != 3 { + t.Fatalf("args = %#v, want three legacy-compatible parameters", args) + } +} + +func buildResolvedAggregationQueryForTest( + t *testing.T, + client *PostgreSQLClient, + pipeline any, +) (string, []any, error) { + t.Helper() + + rawPipeline, options := ResolvePipelineStageOptions(pipeline) + stages, ok := rawPipeline.([]map[string]any) + if !ok { + t.Fatalf("resolved pipeline has type %T, want []map[string]any", rawPipeline) + } + + return client.buildAggregationQuery(stages, options) +} + +func TestAggregationExtendedFilterPrefixRejectsLaterLogicalOperators(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + stages := []map[string]any{ + {"$match": map[string]any{"healthevent.nodename": "node-a"}}, + {"$match": map[string]any{opOr: []any{ + map[string]any{"healthevent.checkname": "legacy-custom-rule"}, + }}}, + } + + _, _, err := buildResolvedAggregationQueryForTest(t, client, WithExtendedFilterPrefix(stages, 1)) + if err == nil { + t.Fatal("extended-only configured operator accepted outside the enabled prefix") + } + if !datastore.IsDeterministicError(err) { + t.Fatalf("configured operator classified as transient: %v", err) + } +} + +func TestAnalyzerAggregationRejectsUnsupportedMatchShapes(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + tests := map[string]any{ + "extended nor": WithExtendedFilters([]map[string]any{ + {"$match": map[string]any{"$nor": []any{ + map[string]any{"healthevent.checkname": "custom-rule"}, + }}}, + }), + "nested nor": WithExtendedFilters([]map[string]any{ + {"$match": map[string]any{opOr: []any{ + map[string]any{"$nor": []any{ + map[string]any{"healthevent.checkname": "custom-rule"}, + }}, + }}}, + }), + "later nor": WithExtendedFilterPrefix([]map[string]any{ + {"$match": map[string]any{"healthevent.nodename": "node-a"}}, + {"$match": map[string]any{"$nor": []any{ + map[string]any{"healthevent.checkname": "custom-rule"}, + }}}, + }, 1), + "later array equality": WithExtendedFilterPrefix([]map[string]any{ + {"$match": map[string]any{"healthevent.nodename": "node-a"}}, + {"$match": map[string]any{"healthevent.errorcode": []any{"94"}}}, + }, 1), + } + + for name, pipeline := range tests { + t.Run(name, func(t *testing.T) { + _, _, err := buildResolvedAggregationQueryForTest(t, client, pipeline) + if err == nil { + t.Fatal("unsupported PostgreSQL $match shape accepted") + } + if !datastore.IsDeterministicError(err) { + t.Fatalf("unsupported shape classified as transient: %v", err) + } + }) + } +} + +func TestPostCountMatchCombinesAllOperators(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + query, args, err := client.buildAggregationQuery([]map[string]any{ + {"$count": "count"}, + {"$match": map[string]any{"count": map[string]any{"$gte": 2, "$lte": 5}}}, + }, PipelineOptions{}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(query, "(document->>'count')::bigint >= $1 AND (document->>'count')::bigint <= $2") { + t.Fatalf("post-count bounds were not combined: %s", query) + } + if !reflect.DeepEqual(args, []any{2, 5}) { + t.Fatalf("args = %#v, want [2 5]", args) + } +} + +func TestPostCountMatch_MultipleFields_OrdersFieldsDeterministically(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + wantQuery := "(document->>'critical')::bigint >= $1 AND (document->>'total')::bigint <= $2" + wantArgs := []any{2, 5} + + for range 20 { + query, args, err := client.buildAggregationQuery([]map[string]any{ + {"$count": "count"}, + {"$match": map[string]any{ + "total": map[string]any{"$lte": 5}, + "critical": map[string]any{"$gte": 2}, + }}, + }, PipelineOptions{}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(query, wantQuery) { + t.Fatalf("post-count fields are not ordered: %s", query) + } + if !reflect.DeepEqual(args, wantArgs) { + t.Fatalf("args = %#v, want %#v", args, wantArgs) + } + } +} + +func TestPostCountMatchRejectsUnsupportedShapes(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + tests := map[string]map[string]any{ + "logical operator": { + opOr: []any{map[string]any{"count": map[string]any{opGTE: 5}}}, + }, + "expression": { + "$expr": map[string]any{opGTE: []any{"$count", 5}}, + }, + "array equality": { + "count": []any{5, 6}, + }, + "array comparison": { + "count": map[string]any{opGTE: []any{5}}, + }, + "unsupported in": { + "count": map[string]any{opIn: []any{5, 6}}, + }, + "invalid exists": { + "count": map[string]any{opExists: "yes"}, + }, + } + + for name, match := range tests { + t.Run(name, func(t *testing.T) { + _, _, err := client.buildAggregationQuery([]map[string]any{ + {"$count": "count"}, + {"$match": match}, + }, PipelineOptions{}) + if err == nil { + t.Fatal("unsupported post-count $match accepted") + } + if !datastore.IsDeterministicError(err) { + t.Fatalf("post-count validation error classified as transient: %v", err) + } + }) + } +} + +func TestPostCountMatchNeverDropsNonEmptyFilter(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + query, _, err := client.buildAggregationQuery([]map[string]any{ + {"$count": "count"}, + {"$match": map[string]any{"count": map[string]any{opGTE: 5}}}, + }, PipelineOptions{}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(query, "count_result WHERE") { + t.Fatalf("post-count filter produced an unfiltered count query: %s", query) + } +} + +func TestExprBuilderClassifiesMalformedShapesDeterministically(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + tests := map[string]any{ + "bad filter condition": map[string]any{ + opEQ: []any{ + map[string]any{"$filter": map[string]any{"input": "$items", "cond": "invalid"}}, + 1, + }, + }, + "bad map expression": map[string]any{ + opEQ: []any{ + map[string]any{"$map": map[string]any{"input": "$items", "in": []any{"invalid"}}}, + 1, + }, + }, + "multiple operators": map[string]any{ + opGT: []any{"$count", 1}, + opLT: []any{"$count", 9}, + }, + } + + for name, expr := range tests { + t.Run(name, func(t *testing.T) { + _, err := client.buildExprCondition(expr) + if err == nil { + t.Fatal("malformed $expr accepted") + } + if !datastore.IsDeterministicError(err) { + t.Fatalf("malformed $expr classified as transient: %v", err) + } + }) + } +} + +func TestUnsupportedAggregationStageIsDeterministic(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + _, _, err := client.buildAggregationQuery([]map[string]any{ + {"$project": map[string]any{"healthevent": 1}}, + }, PipelineOptions{}) + if err == nil { + t.Fatal("unsupported PostgreSQL aggregation stage accepted") + } + if !datastore.IsDeterministicError(err) { + t.Fatalf("unsupported stage classified as transient: %v", err) + } +} + +func TestExtendedEmptyMatchUsesTrueClause(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + query, args, err := client.buildAggregationQuery([]map[string]any{ + {"$match": map[string]any{}}, + }, PipelineOptions{EnableExtendedFilters: true}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(query, "WHERE TRUE") || len(args) != 0 { + t.Fatalf("query = %s, args = %#v", query, args) + } +} + +func TestAggregationRecoveryBoundaryUsesCreatedAtColumn(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + cutoff := time.Date(2026, 8, 29, 12, 0, 0, 0, time.UTC) + + query, args, err := client.buildAggregationQuery([]map[string]any{ + {"$match": map[string]any{"createdAt": map[string]any{"$gt": cutoff}}}, + }, PipelineOptions{EnableExtendedFilters: true}) + if err != nil { + t.Fatalf("buildAggregationQuery() error = %v", err) + } + + if !strings.Contains(query, "created_at > $1") { + t.Fatalf("query does not use typed created_at boundary: %s", query) + } + + if len(args) != 1 || !args[0].(time.Time).Equal(cutoff) { + t.Fatalf("args = %v, want [%s]", args, cutoff) + } +} + +func TestAggregationRecoveryBoundarySupportsNanosecondEventTime(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + + query, _, err := client.buildAggregationQuery([]map[string]any{ + {"$match": map[string]any{ + "$expr": map[string]any{ + "$or": []any{ + map[string]any{ + "$gt": []any{"$healthevent.generatedtimestamp.seconds", int64(100)}, + }, + map[string]any{ + "$and": []any{ + map[string]any{ + "$eq": []any{"$healthevent.generatedtimestamp.seconds", int64(100)}, + }, + map[string]any{ + "$gt": []any{"$healthevent.generatedtimestamp.nanos", int64(250)}, + }, + }, + }, + }, + }, + }}, + }, PipelineOptions{EnableExtendedFilters: true}) + if err != nil { + t.Fatalf("buildAggregationQuery() error = %v", err) + } + + for _, expected := range []string{"generatedTimestamp'->>'seconds')::bigint > 100", + "generatedTimestamp'->>'nanos')::bigint > 250", " OR ", " AND "} { + if !strings.Contains(query, expected) { + t.Fatalf("query does not contain %q: %s", expected, query) + } + } +} + +func TestAggregationSupportsAnalyzerMandatoryLogicalFilter(t *testing.T) { + client := &PostgreSQLClient{table: "health_events"} + + query, args, err := client.buildAggregationQuery([]map[string]any{ + {"$match": map[string]any{ + "healthevent.agent": map[string]any{"$ne": "health-events-analyzer"}, + "healthevent.ishealthy": false, + "$or": []any{ + map[string]any{"healthevent.processingstrategy": int32(1)}, + map[string]any{"healthevent.processingstrategy": int32(2)}, + map[string]any{ + "healthevent.processingstrategy": map[string]any{"$exists": false}, + }, + }, + }}, + }, PipelineOptions{EnableExtendedFilters: true}) + if err != nil { + t.Fatalf("buildAggregationQuery() error = %v", err) + } + + for _, expected := range []string{" OR ", "IS NULL", "document->'healthevent'->>'processingStrategy'"} { + if !strings.Contains(query, expected) { + t.Fatalf("query does not contain %q: %s", expected, query) + } + } + if !strings.Contains(query, "document->'healthevent'->>'agent' IS DISTINCT FROM") { + t.Fatalf("agent exclusion is not null-safe: %s", query) + } + + for _, arg := range args { + if _, isMap := arg.(map[string]any); isMap { + t.Fatalf("SQL argument must not contain a logical-filter map: %#v", args) + } + } + + wantArgs := []any{"1", "2", "health-events-analyzer", false} + if !reflect.DeepEqual(args, wantArgs) { + t.Fatalf("args = %#v, want %#v", args, wantArgs) + } } // TestBuildUpdateClause tests update translation logic @@ -348,7 +897,7 @@ func TestAggregationPipelineConversion(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, _, err := client.buildAggregationQuery(tt.stages) + _, _, err := client.buildAggregationQuery(tt.stages, PipelineOptions{}) if tt.expectError { if err == nil { @@ -455,7 +1004,7 @@ func TestSetWindowFieldsQueryGeneration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - query, _, err := client.buildAggregationQuery(tt.stages) + query, _, err := client.buildAggregationQuery(tt.stages, PipelineOptions{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -860,7 +1409,7 @@ func TestAddFieldsWithNewOperators(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - query, _, err := client.buildAggregationQuery(tt.stages) + query, _, err := client.buildAggregationQuery(tt.stages, PipelineOptions{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -896,7 +1445,7 @@ func TestCountWithPostMatchFilter(t *testing.T) { {"$match": map[string]any{"count": map[string]any{"$gte": 5}}}, } - query, args, err := client.buildAggregationQuery(stages) + query, args, err := client.buildAggregationQuery(stages, PipelineOptions{}) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -953,7 +1502,7 @@ func TestCountWithPostMatchFilter_ZeroCount(t *testing.T) { {"$match": map[string]any{"count": map[string]any{"$gte": 5}}}, } - query, args, err := client.buildAggregationQuery(stages) + query, args, err := client.buildAggregationQuery(stages, PipelineOptions{}) if err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/store-client/pkg/client/postgresql_pipeline_builder.go b/store-client/pkg/client/postgresql_pipeline_builder.go index 606d50c54..aca69429e 100644 --- a/store-client/pkg/client/postgresql_pipeline_builder.go +++ b/store-client/pkg/client/postgresql_pipeline_builder.go @@ -177,6 +177,38 @@ func (b *PostgreSQLPipelineBuilder) BuildProcessableNonFatalUnhealthyInsertsPipe ) } +// BuildAnalyzerHealthEventInsertsPipeline creates the analyzer input pipeline. +// PostgreSQL watches inserts and status updates, matching the existing analyzer +// pipeline behavior while admitting healthy recovery events. +func (b *PostgreSQLPipelineBuilder) BuildAnalyzerHealthEventInsertsPipeline() datastore.Pipeline { + return datastore.ToPipeline( + datastore.D( + datastore.E(opMatch, datastore.D( + datastore.E(fieldOperationType, datastore.D(datastore.E(opIn, datastore.A(opTypeInsert, "update")))), + datastore.E("fullDocument.healthevent.agent", datastore.D(datastore.E(opNE, "health-events-analyzer"))), + datastore.E("$or", datastore.A( + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + int32(protos.ProcessingStrategy_UNSPECIFIED), + )), + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), + )), + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + int32(protos.ProcessingStrategy_STORE_AND_ANALYSE), + )), + datastore.D(datastore.E( + "fullDocument.healthevent.processingstrategy", + datastore.D(datastore.E("$exists", false)), + )), + )), + )), + ), + ) +} + // BuildQuarantinedAndDrainedNodesPipeline creates a pipeline for remediation-ready nodes // Similar to BuildNodeQuarantineStatusPipeline, this supports both UPDATE and INSERT operations // to be defensive against PostgreSQL trigger edge cases. diff --git a/store-client/pkg/datastore/errors.go b/store-client/pkg/datastore/errors.go index c92831419..e2b1caf3e 100644 --- a/store-client/pkg/datastore/errors.go +++ b/store-client/pkg/datastore/errors.go @@ -140,6 +140,53 @@ func IsRetryableError(err error) bool { return false } +// IsDeterministicError reports datastore failures that replaying the same +// event cannot repair. Connection, timeout, permission, and unknown query +// failures remain retryable by the caller. +func IsDeterministicError(err error) bool { + var datastoreErr *DatastoreError + if !errors.As(err, &datastoreErr) { + return false + } + + switch datastoreErr.Type { + case ErrorTypeValidation, ErrorTypeSerialization, ErrorTypeConversion, ErrorTypeConfiguration: + return true + case ErrorTypeQuery: + return isDeterministicQueryError(datastoreErr.Cause) + case ErrorTypeConnection, ErrorTypeAuthentication, ErrorTypeTimeout, ErrorTypeCertificate, + ErrorTypeInsert, ErrorTypeUpdate, ErrorTypeDelete, ErrorTypeTransaction, + ErrorTypeDocumentNotFound, ErrorTypeProviderNotFound, ErrorTypeInvalidProvider, + ErrorTypeChangeStream, ErrorTypeResumeToken, ErrorTypeUnknown: + return false + default: + return false + } +} + +func isDeterministicQueryError(cause error) bool { + if cause == nil { + return true + } + + var postgresErr interface{ SQLState() string } + if !errors.As(cause, &postgresErr) { + return false + } + + code := postgresErr.SQLState() + if len(code) >= 2 && code[:2] == "22" { + return true + } + + switch code { + case "42601", "42703", "42804", "42883": + return true + default: + return false + } +} + // IsNotFoundError checks if the error indicates a document was not found func IsNotFoundError(err error) bool { if datastoreErr, ok := errors.AsType[*DatastoreError](err); ok { diff --git a/store-client/pkg/datastore/errors_deterministic_test.go b/store-client/pkg/datastore/errors_deterministic_test.go new file mode 100644 index 000000000..d61121e12 --- /dev/null +++ b/store-client/pkg/datastore/errors_deterministic_test.go @@ -0,0 +1,72 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package datastore + +import ( + "errors" + "testing" + + "github.com/lib/pq" +) + +func TestIsDeterministicError(t *testing.T) { + for name, test := range map[string]struct { + err error + want bool + }{ + "validation": { + err: NewValidationError(ProviderPostgreSQL, "bad pipeline", nil), + want: true, + }, + "postgres data exception": { + err: NewQueryError(ProviderPostgreSQL, "bad row", &pq.Error{Code: "22P02"}), + want: true, + }, + "postgres syntax error": { + err: NewQueryError(ProviderPostgreSQL, "bad SQL", &pq.Error{Code: "42601"}), + want: true, + }, + "postgres undefined table": { + err: NewQueryError(ProviderPostgreSQL, "table not provisioned yet", &pq.Error{Code: "42P01"}), + }, + "postgres permission error": { + err: NewQueryError(ProviderPostgreSQL, "permission", &pq.Error{Code: "42501"}), + }, + "unknown query failure": { + err: NewQueryError(ProviderMongoDB, "query failed", errors.New("server unavailable")), + }, + "connection failure": { + err: NewConnectionError(ProviderPostgreSQL, "offline", errors.New("refused")), + }, + "plain error": {err: errors.New("plain")}, + } { + t.Run(name, func(t *testing.T) { + if got := IsDeterministicError(test.err); got != test.want { + t.Fatalf("IsDeterministicError() = %t, want %t", got, test.want) + } + }) + } +} + +func TestDeterministicPostgreSQLQuerySQLStates(t *testing.T) { + for _, code := range []string{"22000", "42601", "42703", "42804", "42883"} { + t.Run(code, func(t *testing.T) { + err := NewQueryError(ProviderPostgreSQL, "invalid deterministic query", &pq.Error{Code: pq.ErrorCode(code)}) + if !IsDeterministicError(err) { + t.Fatalf("SQLSTATE %s classified as transient", code) + } + }) + } +} diff --git a/store-client/pkg/datastore/providers/mongodb/adapter.go b/store-client/pkg/datastore/providers/mongodb/adapter.go index 0b561b295..08d381857 100644 --- a/store-client/pkg/datastore/providers/mongodb/adapter.go +++ b/store-client/pkg/datastore/providers/mongodb/adapter.go @@ -152,6 +152,8 @@ func (a *AdaptedMongoStore) GetCollectionClient() client.CollectionClient { // CreateChangeStreamWatcher creates a change stream watcher func (a *AdaptedMongoStore) CreateChangeStreamWatcher(ctx context.Context, clientName string, pipeline any) (datastore.ChangeStreamWatcher, error) { + pipeline, _ = client.ResolvePipelineOptions(pipeline) + // Use our existing factory to create a change stream watcher // Note: Token configuration is loaded from environment variables by the factory // via config.TokenConfigFromEnv(clientName). To customize token collection, diff --git a/store-client/pkg/datastore/providers/postgresql/analyzer_pipeline_test.go b/store-client/pkg/datastore/providers/postgresql/analyzer_pipeline_test.go new file mode 100644 index 000000000..4dbd3c05f --- /dev/null +++ b/store-client/pkg/datastore/providers/postgresql/analyzer_pipeline_test.go @@ -0,0 +1,112 @@ +// Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package postgresql + +import ( + "testing" + + "github.com/nvidia/nvsentinel/data-models/pkg/protos" + "github.com/nvidia/nvsentinel/store-client/pkg/client" + "github.com/nvidia/nvsentinel/store-client/pkg/datastore" +) + +func TestAnalyzerPipelineAdmitsRecoveryEvents(t *testing.T) { + filter, err := NewPipelineFilter( + client.WithExtendedFilters( + client.NewPostgreSQLPipelineBuilder().BuildAnalyzerHealthEventInsertsPipeline(), + ), + ) + if err != nil { + t.Fatalf("NewPipelineFilter() error = %v", err) + } + + tests := []struct { + name string + operation string + agent string + strategy any + isHealthy bool + want bool + }{ + {name: "healthy insert", operation: "insert", agent: "syslog-health-monitor", + strategy: int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), isHealthy: true, want: true}, + {name: "unhealthy insert", operation: "insert", agent: "syslog-health-monitor", + strategy: int32(protos.ProcessingStrategy_STORE_AND_ANALYSE), want: true}, + {name: "healthy update", operation: "update", agent: "syslog-health-monitor", + strategy: int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), isHealthy: true, want: true}, + {name: "legacy event without strategy", operation: "insert", agent: "custom-monitor", + strategy: nil, isHealthy: true, want: true}, + {name: "explicit unspecified strategy", operation: "insert", agent: "custom-monitor", + strategy: int32(protos.ProcessingStrategy_UNSPECIFIED), isHealthy: true, want: true}, + {name: "legacy event without agent", operation: "insert", + strategy: int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), isHealthy: true, want: true}, + {name: "analyzer output", operation: "insert", agent: "health-events-analyzer", + strategy: int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), isHealthy: true, want: false}, + {name: "store only", operation: "insert", agent: "syslog-health-monitor", + strategy: int32(protos.ProcessingStrategy_STORE_ONLY), isHealthy: true, want: false}, + {name: "delete", operation: "delete", agent: "syslog-health-monitor", + strategy: int32(protos.ProcessingStrategy_EXECUTE_REMEDIATION), isHealthy: true, want: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + healthEvent := map[string]any{ + "ishealthy": test.isHealthy, + } + if test.agent != "" { + healthEvent["agent"] = test.agent + } + if test.strategy != nil { + healthEvent["processingstrategy"] = test.strategy + } + + event := datastore.EventWithToken{Event: datastore.Event{ + "operationType": test.operation, + "fullDocument": map[string]any{ + "healthevent": healthEvent, + }, + }} + + if got := filter.MatchesEvent(event); got != test.want { + t.Fatalf("MatchesEvent() = %t, want %t", got, test.want) + } + }) + } +} + +func TestExtendedFiltersDoNotChangeFaultQuarantineAdmission(t *testing.T) { + pipeline := client.NewPostgreSQLPipelineBuilder().BuildProcessableHealthEventInsertsPipeline() + legacyFilter, err := NewPipelineFilter(pipeline) + if err != nil { + t.Fatal(err) + } + extendedFilter, err := NewPipelineFilter(client.WithExtendedFilters(pipeline)) + if err != nil { + t.Fatal(err) + } + + event := datastore.EventWithToken{Event: datastore.Event{ + "operationType": "insert", + "fullDocument": map[string]any{ + "healthevent": map[string]any{}, + }, + }} + if legacyFilter.MatchesEvent(event) { + t.Fatal("unscoped pipeline unexpectedly changed fault-quarantine admission") + } + if !extendedFilter.MatchesEvent(event) { + t.Fatal("extended pipeline did not admit a legacy event") + } +} diff --git a/store-client/pkg/datastore/providers/postgresql/database_client.go b/store-client/pkg/datastore/providers/postgresql/database_client.go index 3399e231d..c723b1ce0 100644 --- a/store-client/pkg/datastore/providers/postgresql/database_client.go +++ b/store-client/pkg/datastore/providers/postgresql/database_client.go @@ -931,6 +931,10 @@ func (c *PostgreSQLDatabaseClient) Find( // Apply options if options != nil { + if options.Sort != nil { + query += convertMongoSortToSQL(options.Sort) + } + if options.Limit != nil && *options.Limit > 0 { query += fmt.Sprintf(" LIMIT %d", *options.Limit) } @@ -1038,9 +1042,6 @@ func (c *PostgreSQLDatabaseClient) NewChangeStreamWatcher( // 1. Server-side: SQL WHERE clause (built from w.pipeline in fetchNewChanges) // 2. Application-side: PipelineFilter (handles edge cases SQL can't express) if pipeline != nil { - // Store raw pipeline for SQL filter building - watcher.pipeline = pipeline - // Create application-side filter as fallback pipelineFilter, err := NewPipelineFilter(pipeline) if err != nil { @@ -1048,6 +1049,9 @@ func (c *PostgreSQLDatabaseClient) NewChangeStreamWatcher( } else { watcher.pipelineFilter = pipelineFilter } + + // Store the raw pipeline for SQL filter building after consuming options. + watcher.pipeline, _ = client.ResolvePipelineOptions(pipeline) } // Return the adapter that implements client.ChangeStreamWatcher diff --git a/store-client/pkg/datastore/providers/postgresql/database_client_test.go b/store-client/pkg/datastore/providers/postgresql/database_client_test.go index e2bfc7d53..a4475a209 100644 --- a/store-client/pkg/datastore/providers/postgresql/database_client_test.go +++ b/store-client/pkg/datastore/providers/postgresql/database_client_test.go @@ -15,10 +15,14 @@ package postgresql import ( + "context" + "regexp" "strings" "testing" "time" + "github.com/DATA-DOG/go-sqlmock" + "github.com/nvidia/nvsentinel/store-client/pkg/client" "github.com/nvidia/nvsentinel/store-client/pkg/query" ) @@ -225,3 +229,28 @@ func TestFindOneFilterGeneration(t *testing.T) { }) } } + +func TestFindAppliesDescendingCreatedAtSort(t *testing.T) { + db, sqlMock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New() error = %v", err) + } + defer db.Close() + + databaseClient := NewPostgreSQLDatabaseClient(db, "HealthEvents") + sqlMock.ExpectQuery(regexp.QuoteMeta( + "SELECT document FROM health_events WHERE event_type = $1 ORDER BY created_at DESC", + )).WithArgs("DerivedCondition").WillReturnRows(sqlmock.NewRows([]string{"document"})) + + cursor, err := databaseClient.Find(context.Background(), map[string]any{ + "event_type": "DerivedCondition", + }, &client.FindOptions{Sort: map[string]any{"createdAt": -1}}) + if err != nil { + t.Fatalf("Find() error = %v", err) + } + defer cursor.Close(context.Background()) + + if err := sqlMock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectation: %v", err) + } +} diff --git a/store-client/pkg/datastore/providers/postgresql/datastore.go b/store-client/pkg/datastore/providers/postgresql/datastore.go index ea5d83133..169b5dc04 100644 --- a/store-client/pkg/datastore/providers/postgresql/datastore.go +++ b/store-client/pkg/datastore/providers/postgresql/datastore.go @@ -17,6 +17,7 @@ package postgresql import ( "context" "database/sql" + "errors" "fmt" "log/slog" "strings" @@ -38,6 +39,8 @@ type PostgreSQLDataStore struct { connString string // Connection string for creating LISTEN connections maintenanceEventStore datastore.MaintenanceEventStore healthEventStore datastore.HealthEventStore + runtimeUpgradeCancel context.CancelFunc + runtimeUpgradeDone <-chan struct{} } // NewPostgreSQLStore creates a new PostgreSQL datastore @@ -98,6 +101,7 @@ func NewPostgreSQLStore(ctx context.Context, config datastore.DataStoreConfig) ( } store.maintenanceEventStore = NewPostgreSQLMaintenanceEventStore(db) store.healthEventStore = NewPostgreSQLHealthEventStore(db) + store.startRuntimeUpgrades() slog.Info("Successfully connected to PostgreSQL database", "host", config.Connection.Host) @@ -121,6 +125,18 @@ func (p *PostgreSQLDataStore) Ping(ctx context.Context) error { // Close closes the database connection func (p *PostgreSQLDataStore) Close(ctx context.Context) error { + if p.runtimeUpgradeCancel != nil { + p.runtimeUpgradeCancel() + } + + if p.runtimeUpgradeDone != nil { + select { + case <-p.runtimeUpgradeDone: + case <-ctx.Done(): + return errors.Join(ctx.Err(), p.db.Close()) + } + } + return p.db.Close() } @@ -154,6 +170,7 @@ func (p *PostgreSQLDataStore) NewChangeStreamWatcher( } pipelineFilter := buildPipelineFilter(pipeline, tableName, clientName) + pipeline, _ = client.ResolvePipelineOptions(pipeline) // Convert PascalCase table name to snake_case for PostgreSQL compatibility snakeCaseTableName := toSnakeCase(tableName) @@ -332,6 +349,88 @@ var recoveryIndexStatements = []string{ `document->'healtheventstatus'->>'faultquarantinerecovery' = '')`, } +type runtimeUpgradeIndex struct { + name string + createStatement string + dropStatement string +} + +// runtimeUpgradeIndexes are checked on every datastore startup so existing +// databases receive indexes added after initial Helm provisioning. +var runtimeUpgradeIndexes = []runtimeUpgradeIndex{ + { + name: "idx_health_events_analyzer_lookup", + createStatement: `CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_health_events_analyzer_lookup ` + + `ON health_events (node_name, event_type, created_at DESC, ` + + `(document->'healthevent'->>'agent'))`, + dropStatement: `DROP INDEX CONCURRENTLY IF EXISTS idx_health_events_analyzer_lookup`, + }, +} + +const runtimeUpgradeIndexValidityQuery = `SELECT idx.indisvalid + FROM pg_catalog.pg_index AS idx + WHERE idx.indexrelid = pg_catalog.to_regclass($1)` + +// startRuntimeUpgrades applies performance migrations outside the datastore +// startup critical path. Close cancels and joins the task before closing the +// connection pool; an interrupted concurrent index build is retried on the +// next startup by runRuntimeUpgrades. +func (p *PostgreSQLDataStore) startRuntimeUpgrades() { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + + p.runtimeUpgradeCancel = cancel + p.runtimeUpgradeDone = done + + go func() { + defer close(done) + + runRuntimeUpgrades(ctx, p.db) + }() +} + +func warnRuntimeUpgradeError(ctx context.Context, message, index string, err error) { + if ctx.Err() != nil { + return + } + + slog.WarnContext(ctx, message, "index", index, "error", err) +} + +// runRuntimeUpgrades executes each statement directly through the connection +// pool, outside an explicit transaction block. Do not wrap this function in +// BeginTx: PostgreSQL rejects CREATE INDEX CONCURRENTLY inside a transaction +// block. +func runRuntimeUpgrades(ctx context.Context, db *sql.DB) { + for _, index := range runtimeUpgradeIndexes { + var valid bool + + err := db.QueryRowContext(ctx, runtimeUpgradeIndexValidityQuery, index.name).Scan(&valid) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + warnRuntimeUpgradeError(ctx, "Failed to inspect PostgreSQL runtime upgrade index", index.name, err) + + continue + } + + if err == nil && valid { + continue + } + + if err == nil { + if _, err := db.ExecContext(ctx, index.dropStatement); err != nil { + warnRuntimeUpgradeError(ctx, + "Failed to remove invalid PostgreSQL runtime upgrade index", index.name, err) + + continue + } + } + + if _, err := db.ExecContext(ctx, index.createStatement); err != nil { + warnRuntimeUpgradeError(ctx, "Failed to apply PostgreSQL runtime upgrade", index.name, err) + } + } +} + func createTables(ctx context.Context, db *sql.DB) error { schemas := []string{ `CREATE EXTENSION IF NOT EXISTS "uuid-ossp"`, diff --git a/store-client/pkg/datastore/providers/postgresql/datastore_test.go b/store-client/pkg/datastore/providers/postgresql/datastore_test.go index 87aff2aec..4522ad5f6 100644 --- a/store-client/pkg/datastore/providers/postgresql/datastore_test.go +++ b/store-client/pkg/datastore/providers/postgresql/datastore_test.go @@ -16,9 +16,12 @@ package postgresql import ( "context" + "database/sql" "fmt" + "regexp" "strings" "testing" + "time" "github.com/DATA-DOG/go-sqlmock" "github.com/stretchr/testify/assert" @@ -190,6 +193,37 @@ func TestPostgreSQLDataStore_Close(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } +func TestPostgreSQLDataStore_RuntimeUpgrades_RunInBackgroundAndStopOnClose(t *testing.T) { + originalIndexes := runtimeUpgradeIndexes + runtimeUpgradeIndexes = []runtimeUpgradeIndex{{ + name: "delayed_index", + createStatement: "delayed concurrent index", + }} + t.Cleanup(func() { runtimeUpgradeIndexes = originalIndexes }) + + db, mock, err := sqlmock.New() + require.NoError(t, err) + + mock.ExpectQuery(regexp.QuoteMeta(runtimeUpgradeIndexValidityQuery)). + WithArgs("delayed_index"). + WillDelayFor(5 * time.Second). + WillReturnError(sql.ErrNoRows) + + ds := &PostgreSQLDataStore{db: db} + startedAt := time.Now() + ds.startRuntimeUpgrades() + require.Less(t, time.Since(startedAt), time.Second) + + require.Eventually(t, func() bool { + return mock.ExpectationsWereMet() == nil + }, time.Second, 10*time.Millisecond) + mock.ExpectClose() + closeCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, ds.Close(closeCtx)) + require.NoError(t, mock.ExpectationsWereMet()) +} + func TestPostgreSQLDataStore_Ping(t *testing.T) { db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) require.NoError(t, err) @@ -254,6 +288,102 @@ func TestRecoveryIndexesIncludePartialPendingEventCursor(t *testing.T) { "document->'healtheventstatus'->>'faultquarantinerecovery' IS NULL") } +func TestRuntimeUpgradeIndexes_ExistingDatabase_IncludesAnalyzerLookup(t *testing.T) { + var analyzerIndex runtimeUpgradeIndex + for _, index := range runtimeUpgradeIndexes { + if index.name == "idx_health_events_analyzer_lookup" { + analyzerIndex = index + + break + } + } + + require.NotEmpty(t, analyzerIndex.name) + assert.Contains(t, analyzerIndex.createStatement, "CREATE INDEX CONCURRENTLY IF NOT EXISTS") + assert.Contains(t, analyzerIndex.createStatement, "ON health_events (node_name, event_type, created_at DESC") + assert.Contains(t, analyzerIndex.createStatement, "document->'healthevent'->>'agent'") + assert.Equal(t, "DROP INDEX CONCURRENTLY IF EXISTS idx_health_events_analyzer_lookup", analyzerIndex.dropStatement) +} + +func TestRunRuntimeUpgrades_ConcurrentIndex_ExecutesOutsideTransaction(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + for _, index := range runtimeUpgradeIndexes { + mock.ExpectQuery(regexp.QuoteMeta(runtimeUpgradeIndexValidityQuery)). + WithArgs(index.name). + WillReturnError(sql.ErrNoRows) + mock.ExpectExec(regexp.QuoteMeta(index.createStatement)).WillReturnResult(sqlmock.NewResult(0, 0)) + } + + runRuntimeUpgrades(context.Background(), db) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRunRuntimeUpgrades_OneStatementFails_ContinuesRemainingStatements(t *testing.T) { + originalIndexes := runtimeUpgradeIndexes + runtimeUpgradeIndexes = []runtimeUpgradeIndex{ + {name: "first_index", createStatement: "first concurrent index"}, + {name: "second_index", createStatement: "second concurrent index"}, + } + t.Cleanup(func() { runtimeUpgradeIndexes = originalIndexes }) + + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + for _, index := range runtimeUpgradeIndexes { + mock.ExpectQuery(regexp.QuoteMeta(runtimeUpgradeIndexValidityQuery)). + WithArgs(index.name). + WillReturnError(sql.ErrNoRows) + if index.name == "first_index" { + mock.ExpectExec(regexp.QuoteMeta(index.createStatement)).WillReturnError(assert.AnError) + } else { + mock.ExpectExec(regexp.QuoteMeta(index.createStatement)).WillReturnResult(sqlmock.NewResult(0, 0)) + } + } + + runRuntimeUpgrades(context.Background(), db) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRunRuntimeUpgrades_InvalidIndex_RecreatesConcurrently(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + index := runtimeUpgradeIndexes[0] + mock.ExpectQuery(regexp.QuoteMeta(runtimeUpgradeIndexValidityQuery)). + WithArgs(index.name). + WillReturnRows(sqlmock.NewRows([]string{"indisvalid"}).AddRow(false)) + mock.ExpectExec(regexp.QuoteMeta(index.dropStatement)).WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta(index.createStatement)).WillReturnResult(sqlmock.NewResult(0, 0)) + + runRuntimeUpgrades(context.Background(), db) + require.NoError(t, mock.ExpectationsWereMet()) +} + +func TestRunRuntimeUpgrades_ValidIndex_SkipsMigration(t *testing.T) { + db, mock, err := sqlmock.New() + require.NoError(t, err) + defer db.Close() + + index := runtimeUpgradeIndexes[0] + mock.ExpectQuery(regexp.QuoteMeta(runtimeUpgradeIndexValidityQuery)). + WithArgs(index.name). + WillReturnRows(sqlmock.NewRows([]string{"indisvalid"}).AddRow(true)) + // sqlmock has no negative expectations. Queue both destructive statements + // and require them to remain unmet so removing the valid-index guard fails + // this test instead of being hidden by the helper's best-effort error path. + mock.ExpectExec(regexp.QuoteMeta(index.dropStatement)).WillReturnResult(sqlmock.NewResult(0, 0)) + mock.ExpectExec(regexp.QuoteMeta(index.createStatement)).WillReturnResult(sqlmock.NewResult(0, 0)) + + runRuntimeUpgrades(context.Background(), db) + expectationsErr := mock.ExpectationsWereMet() + require.ErrorContains(t, expectationsErr, "ExpectedExec") +} + func TestPostgreSQLDataStore_Provider(t *testing.T) { db, _, err := sqlmock.New() require.NoError(t, err) diff --git a/store-client/pkg/datastore/providers/postgresql/pipeline_filter.go b/store-client/pkg/datastore/providers/postgresql/pipeline_filter.go index e5f58992c..4db731c10 100644 --- a/store-client/pkg/datastore/providers/postgresql/pipeline_filter.go +++ b/store-client/pkg/datastore/providers/postgresql/pipeline_filter.go @@ -19,13 +19,15 @@ import ( "log/slog" "strings" + "github.com/nvidia/nvsentinel/store-client/pkg/client" "github.com/nvidia/nvsentinel/store-client/pkg/datastore" ) // PipelineFilter filters events based on MongoDB-style aggregation pipeline // This allows PostgreSQL to emulate MongoDB's pipeline filtering at the application level type PipelineFilter struct { - stages []filterStage + stages []filterStage + extendedFilters bool } // filterStage represents a single stage in the pipeline (currently only $match is supported) @@ -35,12 +37,15 @@ type filterStage struct { // NewPipelineFilter creates a new pipeline filter from a MongoDB-style pipeline func NewPipelineFilter(pipeline any) (*PipelineFilter, error) { + pipeline, extendedFilters := client.ResolvePipelineOptions(pipeline) + if pipeline == nil { return nil, nil } filter := &PipelineFilter{ - stages: make([]filterStage, 0), + stages: make([]filterStage, 0), + extendedFilters: extendedFilters, } // Handle different pipeline types @@ -173,12 +178,43 @@ func (f *PipelineFilter) matchesCondition(event map[string]any, key string, expe return result default: // Handle field path matching (e.g., "operationType", "fullDocument.healtheventstatus.faultremediated.value") - actualValue := f.getFieldValue(event, key) + if !f.extendedFilters { + return f.matchesValue(f.getFieldValue(event, key), expectedValue) + } - result := f.matchesValue(actualValue, expectedValue) + actualValue, present := f.getFieldValueWithPresence(event, key) - return result + return f.matchesFieldCondition(actualValue, present, expectedValue) + } +} + +func (f *PipelineFilter) matchesFieldCondition(actualValue any, present bool, expectedValue any) bool { + expectedMap, ok := filterMap(expectedValue) + if !ok { + return f.matchesValue(actualValue, expectedValue) + } + + expectedExists, hasExists := expectedMap[opExists] + if !hasExists { + return f.matchesValue(actualValue, expectedValue) + } + + if !matchesExists(present, expectedExists) { + return false + } + + if len(expectedMap) == 1 { + return true + } + + remaining := make(map[string]any, len(expectedMap)-1) + for operator, value := range expectedMap { + if operator != opExists { + remaining[operator] = value + } } + + return f.matchesValue(actualValue, remaining) } // matchesOr handles $or conditions @@ -308,6 +344,35 @@ func (f *PipelineFilter) matchesMapValue(actualValue any, expectedMap map[string // matchesOperators processes MongoDB operator expressions func (f *PipelineFilter) matchesOperators(actualValue any, operators map[string]any) bool { + if !f.extendedFilters { + return f.matchesLegacyOperators(actualValue, operators) + } + + for op, opValue := range operators { + var matches bool + + switch op { + case opIn: + matches = f.matchesIn(actualValue, opValue) + case opNe: + matches = !f.matchesEqual(actualValue, opValue) + case opEq: + matches = f.matchesEqual(actualValue, opValue) + case opExists: + matches = matchesExists(actualValue != nil, opValue) + default: + matches = f.matchesOrderedOperator(actualValue, op, opValue) + } + + if !matches { + return false + } + } + + return true +} + +func (f *PipelineFilter) matchesLegacyOperators(actualValue any, operators map[string]any) bool { for op, opValue := range operators { switch op { case opIn: @@ -326,6 +391,7 @@ func (f *PipelineFilter) matchesOperators(actualValue any, operators map[string] return f.matchesLessThanOrEqual(actualValue, opValue) default: slog.Warn("Unsupported operator", "operator", op) + return false } } @@ -333,6 +399,52 @@ func (f *PipelineFilter) matchesOperators(actualValue any, operators map[string] return true } +func filterMap(value any) (map[string]any, bool) { + if mapped, ok := value.(map[string]any); ok { + return mapped, true + } + + document, ok := value.(datastore.Document) + if !ok { + return nil, false + } + + mapped := make(map[string]any, len(document)) + for _, element := range document { + mapped[element.Key] = element.Value + } + + return mapped, true +} + +func matchesExists(present bool, expectedValue any) bool { + expected, ok := expectedValue.(bool) + if !ok { + slog.Warn("$exists operand is not boolean", "type", fmt.Sprintf("%T", expectedValue)) + + return false + } + + return present == expected +} + +func (f *PipelineFilter) matchesOrderedOperator(actualValue any, operator string, expectedValue any) bool { + switch operator { + case opGt: + return f.matchesGreaterThan(actualValue, expectedValue) + case opGte: + return f.matchesGreaterThanOrEqual(actualValue, expectedValue) + case opLt: + return f.matchesLessThan(actualValue, expectedValue) + case opLte: + return f.matchesLessThanOrEqual(actualValue, expectedValue) + default: + slog.Warn("Unsupported operator", "operator", operator) + + return false + } +} + // matchesNestedFields checks if actualValue (as a map) contains expected fields func (f *PipelineFilter) matchesNestedFields(actualValue any, expectedFields map[string]any) bool { actualMap, ok := actualValue.(map[string]any) @@ -521,6 +633,12 @@ func (f *PipelineFilter) matchesLessThanOrEqual(actual, expected any) bool { // e.g., "operationType" or "fullDocument.healthevent.isfatal" // Performs case-insensitive key matching to handle MongoDB (lowercase) vs PostgreSQL (camelCase) differences func (f *PipelineFilter) getFieldValue(event map[string]any, fieldPath string) any { + value, _ := f.getFieldValueWithPresence(event, fieldPath) + + return value +} + +func (f *PipelineFilter) getFieldValueWithPresence(event map[string]any, fieldPath string) (any, bool) { parts := strings.Split(fieldPath, ".") current := any(event) @@ -548,14 +666,14 @@ func (f *PipelineFilter) getFieldValue(event map[string]any, fieldPath string) a } if !found { - return nil // Path doesn't exist + return nil, false } } else { - return nil // Path doesn't exist + return nil, false } } - return current + return current, true } // toFloat64 converts various numeric types to float64 diff --git a/store-client/pkg/datastore/providers/postgresql/pipeline_filter_test.go b/store-client/pkg/datastore/providers/postgresql/pipeline_filter_test.go index caceee6ac..dfdbee5bf 100644 --- a/store-client/pkg/datastore/providers/postgresql/pipeline_filter_test.go +++ b/store-client/pkg/datastore/providers/postgresql/pipeline_filter_test.go @@ -17,9 +17,101 @@ package postgresql import ( "testing" + "github.com/nvidia/nvsentinel/store-client/pkg/client" "github.com/nvidia/nvsentinel/store-client/pkg/datastore" ) +func TestRecoveryPipelineOperators(t *testing.T) { + filter := &PipelineFilter{extendedFilters: true} + + if !matchesExists(true, true) || !matchesExists(false, false) { + t.Fatal("$exists did not match field presence") + } + if matchesExists(true, false) || matchesExists(false, true) || matchesExists(true, "true") { + t.Fatal("$exists accepted a mismatched or invalid operand") + } + if !filter.matchesOperators(2, map[string]any{opGt: 1, opLt: 3}) || + filter.matchesOperators(2, map[string]any{opGt: 1, opLt: 2}) { + t.Fatal("multiple comparison operators were not combined with AND") + } + + for operator, expected := range map[string]bool{ + opGt: true, opGte: true, opLt: false, opLte: false, + } { + if actual := filter.matchesOrderedOperator(2, operator, 1); actual != expected { + t.Fatalf("matchesOrderedOperator(%q) = %v, want %v", operator, actual, expected) + } + } + if filter.matchesOrderedOperator(2, "$unsupported", 1) { + t.Fatal("unsupported ordered operator matched") + } + if !filter.matchesOperators(1, map[string]any{}) { + t.Fatal("empty operator set should match") + } + if filter.matchesOperators(1, map[string]any{"$unsupported": 1}) { + t.Fatal("unsupported operator matched") + } +} + +func TestExistsDistinguishesMissingAndExplicitNull(t *testing.T) { + for name, test := range map[string]struct { + event map[string]any + exists bool + expected bool + }{ + "missing matches false": { + event: map[string]any{"fullDocument": map[string]any{"healthevent": map[string]any{}}}, + expected: true, + }, + "null does not match false": { + event: map[string]any{"fullDocument": map[string]any{"healthevent": map[string]any{ + "processingstrategy": nil, + }}}, + }, + "null matches true": { + event: map[string]any{"fullDocument": map[string]any{"healthevent": map[string]any{ + "processingstrategy": nil, + }}}, + exists: true, + expected: true, + }, + } { + t.Run(name, func(t *testing.T) { + filter, err := NewPipelineFilter(client.WithExtendedFilters([]any{map[string]any{"$match": map[string]any{ + "fullDocument.healthevent.processingstrategy": map[string]any{"$exists": test.exists}, + }}})) + if err != nil { + t.Fatal(err) + } + if got := filter.MatchesEvent(datastore.EventWithToken{Event: test.event}); got != test.expected { + t.Fatalf("MatchesEvent() = %t, want %t", got, test.expected) + } + }) + } +} + +func TestExistsCombinesWithOtherOperators(t *testing.T) { + filter, err := NewPipelineFilter(client.WithExtendedFilters([]any{map[string]any{"$match": map[string]any{ + "fullDocument.count": map[string]any{"$exists": true, "$gt": 3}, + }}})) + if err != nil { + t.Fatal(err) + } + + for _, test := range []struct { + event map[string]any + want bool + }{ + {event: map[string]any{"fullDocument": map[string]any{}}, want: false}, + {event: map[string]any{"fullDocument": map[string]any{"count": 2}}, want: false}, + {event: map[string]any{"fullDocument": map[string]any{"count": 4}}, want: true}, + } { + if got := filter.MatchesEvent(datastore.EventWithToken{Event: test.event}); got != test.want { + t.Fatalf("MatchesEvent(%v) = %t, want %t", test.event, got, test.want) + } + } +} + func TestGetFieldValue_CaseInsensitive(t *testing.T) { tests := []struct { name string diff --git a/store-client/pkg/datastore/providers/postgresql/sql_filter_builder.go b/store-client/pkg/datastore/providers/postgresql/sql_filter_builder.go index c8e3d8ee2..80fe08d37 100644 --- a/store-client/pkg/datastore/providers/postgresql/sql_filter_builder.go +++ b/store-client/pkg/datastore/providers/postgresql/sql_filter_builder.go @@ -599,11 +599,7 @@ func (b *SQLFilterBuilder) handleNeBool(jsonPath string, v bool) (string, error) b.argIndex++ b.args = append(b.args, v) - if v { - return fmt.Sprintf("((%s)::boolean = false OR %s IS NULL)", jsonPath, jsonPath), nil - } - - return fmt.Sprintf("(%s)::boolean = true", jsonPath), nil + return fmt.Sprintf("(%s)::boolean IS DISTINCT FROM $%d", jsonPath, b.argIndex), nil } // handleNeString handles $ne with string value. @@ -611,7 +607,7 @@ func (b *SQLFilterBuilder) handleNeString(jsonPath string, v string) (string, er b.argIndex++ b.args = append(b.args, v) - return fmt.Sprintf("(%s IS NULL OR %s != $%d)", jsonPath, jsonPath, b.argIndex), nil + return fmt.Sprintf("%s IS DISTINCT FROM $%d", jsonPath, b.argIndex), nil } // handleNeDefault handles $ne with default value type. @@ -619,7 +615,7 @@ func (b *SQLFilterBuilder) handleNeDefault(jsonPath string, v any) (string, erro b.argIndex++ b.args = append(b.args, fmt.Sprintf("%v", v)) - return fmt.Sprintf("(%s IS NULL OR %s != $%d)", jsonPath, jsonPath, b.argIndex), nil + return fmt.Sprintf("%s IS DISTINCT FROM $%d", jsonPath, b.argIndex), nil } // handleInOperator handles $in operator. diff --git a/store-client/pkg/datastore/providers/postgresql/sql_filter_builder_test.go b/store-client/pkg/datastore/providers/postgresql/sql_filter_builder_test.go index 185ccafa3..99d6c4170 100644 --- a/store-client/pkg/datastore/providers/postgresql/sql_filter_builder_test.go +++ b/store-client/pkg/datastore/providers/postgresql/sql_filter_builder_test.go @@ -170,8 +170,7 @@ func TestSQLFilterBuilder_NeOperator(t *testing.T) { clause := builder.GetWhereClause() args := builder.GetArgs() - assert.Contains(t, clause, "IS NULL OR") - assert.Contains(t, clause, "!=") + assert.Contains(t, clause, "IS DISTINCT FROM $4") assert.Len(t, args, 1) assert.Equal(t, "HealthCheck", args[0]) }) @@ -191,9 +190,9 @@ func TestSQLFilterBuilder_NeOperator(t *testing.T) { clause := builder.GetWhereClause() - // $ne: true means field is false or missing - assert.Contains(t, clause, "= false") - assert.Contains(t, clause, "IS NULL") + // IS DISTINCT FROM includes both false and missing values. + assert.Contains(t, clause, "::boolean IS DISTINCT FROM $4") + assert.Equal(t, []any{true}, builder.GetArgs()) }) } @@ -404,6 +403,7 @@ func TestSQLFilterBuilder_HealthEventsAnalyzerPipeline(t *testing.T) { // Should filter by agent != "health-events-analyzer" assert.Contains(t, clause, "healthevent'->>'agent'") + assert.Contains(t, clause, "healthevent'->>'agent' IS DISTINCT FROM") // Should filter by operationType using operation column assert.Contains(t, clause, "operation IN")