|
| 1 | +"""The canonical pgwatch_query_info join operand, shared by both guards. |
| 2 | +
|
| 3 | +pgwatch_query_info arrives in sparse bursts, so joining it as an instant vector |
| 4 | +misses its ~5 min lookback and pgss legends degrade to raw labels. Carrying it |
| 5 | +forward over days then admits a second risk: one queryid can hold several |
| 6 | +displayname* series, which duplicates the joined result. Hence: carry forward, |
| 7 | +newest series wins, value normalised back to 1. See #344. |
| 8 | +
|
| 9 | +A plain module rather than conftest, so importing it from another test package |
| 10 | +does not load conftest a second time alongside pytest's own plugin instance. |
| 11 | +""" |
| 12 | +from __future__ import annotations |
| 13 | + |
| 14 | +import re |
| 15 | + |
| 16 | +QUERY_INFO_METRIC = "pgwatch_query_info" |
| 17 | +PROMQL_DURATION = r"(?:\d+[smhdwy])+" |
| 18 | +# Optional label matcher, so a future MR may scope the selector without the |
| 19 | +# guards rejecting it out of hand. |
| 20 | +QUERY_INFO_SELECTOR = QUERY_INFO_METRIC + r"(?!\w)(?:\{[^}]*\})?" |
| 21 | + |
| 22 | +QUERY_INFO_JOIN_OPERAND = re.compile( |
| 23 | + r"\(\s*topk\s+by\s*\(\s*queryid\s*\)\s*\(\s*1\s*,\s*tlast_over_time\(" |
| 24 | + + QUERY_INFO_SELECTOR |
| 25 | + + r"\[(" + PROMQL_DURATION + r")\]\)\)\s*>\s*bool\s+0\)" |
| 26 | +) |
| 27 | +# Any mention of the metric, so the guards can require that every one of them |
| 28 | +# is part of a full join operand rather than only rejecting the bare name. |
| 29 | +QUERY_INFO_ANY_REFERENCE = re.compile(QUERY_INFO_METRIC + r"(?!\w)") |
| 30 | + |
| 31 | +# Grafana's label_values() takes a selector, not an expression, so a queryid |
| 32 | +# picker built on it cannot use the operand and is exempt. |
| 33 | +LABEL_VALUES_CALL = re.compile(r"\blabel_values\s*\([^)]*\)") |
| 34 | + |
| 35 | +# Label matcher blocks, blanked out before scanning for join clauses so an |
| 36 | +# `on(...)` inside a label *value* cannot be mistaken for a real one. |
| 37 | +LABEL_MATCHER_BLOCK = re.compile(r"\{[^{}]*\}") |
| 38 | + |
| 39 | +# What may immediately precede a join operand. The `group_left` form copies |
| 40 | +# labels across and is checked; the `unless` form is set exclusion and needs |
| 41 | +# none. Anything else — `group_right`, or no modifier at all — means the |
| 42 | +# legend's labels never cross the join. |
| 43 | +MATCH_CLAUSE = r"\b(on|ignoring)\b\s*\(([^)]*)\)\s*" |
| 44 | +GROUP_LEFT_BEFORE_OPERAND = re.compile(MATCH_CLAUSE + r"group_left\s*\(([^)]*)\)\s*$") |
| 45 | +_KEYWORD, _MATCH_LABELS, _COPIED_LABELS = 1, 2, 3 |
| 46 | +UNLESS_BEFORE_OPERAND = re.compile(r"\bunless\s+" + MATCH_CLAUSE + r"$") |
| 47 | + |
| 48 | +_DURATION_UNIT_SECONDS = { |
| 49 | + "s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800, "y": 31536000, |
| 50 | +} |
| 51 | + |
| 52 | +# The observed per-series staleness is hours, so anything shorter than an hour |
| 53 | +# would satisfy the "is it wrapped?" guards while reinstating the bug. |
| 54 | +MIN_QUERY_INFO_LOOKBACK_SECONDS = 3600 |
| 55 | + |
| 56 | + |
| 57 | +def promql_duration_seconds(duration: str) -> int: |
| 58 | + """Convert a PromQL duration ('7d', '1d12h') to seconds.""" |
| 59 | + parts = re.findall(r"(\d+)([smhdwy])", duration) |
| 60 | + assert parts, f"unparseable PromQL duration: {duration!r}" |
| 61 | + return sum(int(amount) * _DURATION_UNIT_SECONDS[unit] for amount, unit in parts) |
| 62 | + |
| 63 | + |
| 64 | +def join_operand_problems(expr: str) -> list[str]: |
| 65 | + """Everything wrong with how one expression references pgwatch_query_info.""" |
| 66 | + problems: list[str] = [] |
| 67 | + references = len(QUERY_INFO_ANY_REFERENCE.findall(expr)) |
| 68 | + if not references: |
| 69 | + return problems |
| 70 | + |
| 71 | + operands = QUERY_INFO_JOIN_OPERAND.findall(expr) |
| 72 | + if len(operands) != references: |
| 73 | + problems.append( |
| 74 | + f"{references} reference(s) but {len(operands)} full join operand(s); " |
| 75 | + "every reference must be the canonical carry-forward operand" |
| 76 | + ) |
| 77 | + for duration in operands: |
| 78 | + seconds = promql_duration_seconds(duration) |
| 79 | + if seconds < MIN_QUERY_INFO_LOOKBACK_SECONDS: |
| 80 | + problems.append( |
| 81 | + f"lookback [{duration}] = {seconds}s is below the " |
| 82 | + f"{MIN_QUERY_INFO_LOOKBACK_SECONDS}s staleness floor" |
| 83 | + ) |
| 84 | + return problems |
| 85 | + |
| 86 | + |
| 87 | +def strip_label_values_calls(expr: str) -> str: |
| 88 | + """Blank out label_values(...) spans, keeping offsets intact. |
| 89 | +
|
| 90 | + Grafana's label_values() takes a selector, not an expression, so a queryid |
| 91 | + picker built on it cannot carry the metric forward and is exempt — but only |
| 92 | + the call itself, not everything else in the same expression. |
| 93 | + """ |
| 94 | + return LABEL_VALUES_CALL.sub(lambda m: " " * len(m.group(0)), expr) |
| 95 | + |
| 96 | + |
| 97 | +def group_left_label_problems(expr: str, required_labels: set[str]) -> list[str]: |
| 98 | + """Labels each join must copy across for the legend to resolve. |
| 99 | +
|
| 100 | + The operand can be perfect while `group_left()` copies nothing — or is |
| 101 | + absent entirely — which renders exactly the raw-label failure the operand |
| 102 | + exists to prevent. Checked per join, not per expression, so one correct |
| 103 | + join cannot vouch for another. |
| 104 | + """ |
| 105 | + problems: list[str] = [] |
| 106 | + if not required_labels or not QUERY_INFO_JOIN_OPERAND.search(expr): |
| 107 | + return problems |
| 108 | + |
| 109 | + # Label values can contain anything, including text that looks like a join |
| 110 | + # clause, so blank the matcher blocks before reading the clauses. |
| 111 | + clauses = LABEL_MATCHER_BLOCK.sub(lambda m: " " * len(m.group(0)), expr) |
| 112 | + |
| 113 | + # Iterate the original expression: blanking is length-preserving, so the |
| 114 | + # offsets still line up, but a blanked label matcher would stop the |
| 115 | + # operand matching at all and silently switch this guard off. |
| 116 | + for operand in QUERY_INFO_JOIN_OPERAND.finditer(expr): |
| 117 | + prefix = clauses[: operand.start()].rstrip() |
| 118 | + |
| 119 | + if UNLESS_BEFORE_OPERAND.search(prefix): |
| 120 | + # Set exclusion: the operand only decides membership, so it copies |
| 121 | + # nothing and needs nothing. |
| 122 | + continue |
| 123 | + |
| 124 | + match = GROUP_LEFT_BEFORE_OPERAND.search(prefix) |
| 125 | + if match is None: |
| 126 | + problems.append( |
| 127 | + "a pgwatch_query_info join has neither group_left(...) nor " |
| 128 | + "unless on(...) before it, so the label(s) the legend renders " |
| 129 | + "never cross it: " + ", ".join(sorted(required_labels)) |
| 130 | + ) |
| 131 | + continue |
| 132 | + |
| 133 | + # Labels matched on are already present on the left-hand side. |
| 134 | + # `ignoring(...)` names the opposite set, so it exempts nothing. |
| 135 | + matched_on: set[str] = set() |
| 136 | + if match.group(_KEYWORD) == "on": |
| 137 | + matched_on = { |
| 138 | + label.strip() |
| 139 | + for label in match.group(_MATCH_LABELS).split(",") |
| 140 | + if label.strip() |
| 141 | + } |
| 142 | + copied = { |
| 143 | + label.strip() |
| 144 | + for label in match.group(_COPIED_LABELS).split(",") |
| 145 | + if label.strip() |
| 146 | + } |
| 147 | + missing = sorted(required_labels - copied - matched_on) |
| 148 | + if missing: |
| 149 | + problems.append( |
| 150 | + "group_left does not copy the label(s) the legend renders: " |
| 151 | + + ", ".join(missing) |
| 152 | + ) |
| 153 | + |
| 154 | + return problems |
0 commit comments