Merge branch 'fix/pgwatch-scrape-sample-limit' into 'main'

fix(monitoring): raise pgwatch scrape sample_limit

Closes #343

See merge request postgres-ai/postgresai!404

Author: Sarumyan

config/prometheus/prometheus.yml

@@ -15,7 +15,14 @@ scrape_configs:
     scrape_interval: 30s      # How often to scrape PGWatch
     scrape_timeout: 25s       # Timeout for each scrape (must be < scrape_interval)
     metrics_path: /pgwatch
-    sample_limit: 10000       # Safety net: allows full preset + top-100 pgss while rejecting cardinality explosions
+    # Backstop against a runaway cardinality explosion — NOT a working ceiling.
+    # sample_limit is all-or-nothing: one sample over the cap and the entire
+    # scrape is discarded and this target goes up=0. It never truncates, so a
+    # snug limit costs every pgwatch series at once, silently. A `full` preset
+    # on a large database reaches ~12.7k samples in steady state, which the
+    # previous 10000 rejected outright. The real cardinality control is the
+    # per-metric top-100 cap in config/pgwatch-prometheus/metrics.yml.
+    sample_limit: 50000
 
   # Self-monitoring: Victoria Metrics internal metrics
   # Note: VictoriaMetrics expands percent-brace env vars in this file (not standard Prometheus)
@@ -57,6 +64,6 @@ scrape_configs:
     # Safety net only — the per-metric LIMIT 100 in pgwatch metrics.yml
     # is the primary cardinality control. Each active queryid emits ~8-9
     # labeled samples, so 500 would cap at ~55-60 queryids and silently
-    # drop the whole scrape on any busy cluster. 5000 is half of the
-    # /pgwatch limit and gives ~550 queryids of headroom.
+    # drop the whole scrape on any busy cluster. 5000 gives ~550 queryids
+    # of headroom, well clear of what the top-100 caps can produce.
     sample_limit: 5000

tests/compliance_vectors/test_mr219_monitoring_guards.py

@@ -18,6 +18,19 @@ def _duration_seconds(value):
 
 
 def test_pgwatch_sample_limit_allows_capped_full_preset():
+    """`sample_limit` is all-or-nothing: one sample over the cap and the WHOLE
+    scrape is discarded and the target goes `up=0`. It does not truncate.
+
+    So the limit must sit well above the largest cardinality normal operation
+    can legitimately produce — a `full` metric preset on a large database runs
+    to ~12.7k samples in steady state, which the previous 10000 cap silently
+    rejected in full, taking every pgwatch series with it.
+
+    The real cardinality control is the per-metric top-100 cap in
+    config/pgwatch-prometheus/metrics.yml. `sample_limit` is only the backstop
+    against a runaway explosion, so it keeps an upper bound too — it must not
+    be removed outright.
+    """
     prometheus = yaml.safe_load(
         (PROJECT_ROOT / "config/prometheus/prometheus.yml").read_text()
     )
@@ -26,8 +39,12 @@ def test_pgwatch_sample_limit_allows_capped_full_preset():
         if job["job_name"] == "pgwatch-prometheus"
     )
 
-    assert pgwatch_job["sample_limit"] >= 10000
-    assert pgwatch_job["sample_limit"] < 50000
+    # Headroom over the observed full-preset steady state, not a snug fit:
+    # cardinality grows with the monitored schema, and the failure mode is
+    # total silent loss rather than partial degradation.
+    assert pgwatch_job["sample_limit"] >= 50000
+    # Still a backstop, not an open door.
+    assert pgwatch_job["sample_limit"] <= 200000
 
     query_info_job = next(
         job for job in prometheus["scrape_configs"]