Skip to content

Commit 848b014

Browse files
nflaignazarhussain
nflaig
and
nazarhussain
authored
feat: add standard fast confirmation metrics (#9552)
**Motivation** Implement the standard fast confirmation metrics defined in ethereum/beacon-metrics#22 **Description** Adds the four standard FCR metrics alongside the existing non-standard `lodestar_*` metrics (kept as-is, even where they overlap): - `beacon_fast_confirmation_slot` — slot of the most recent confirmed block - `beacon_fast_confirmation_reorgs_total` — confirmed block no longer an ancestor of head - `beacon_fast_confirmation_fallbacks_total` — revert of the confirmed block to finalized - `beacon_fast_confirmation_restarts_total` — restart from the observed unrealized justified checkpoint The per-rule `reason` is overwritten by later rules in the pipeline (e.g. a restart is followed by advancing to the latest confirmed descendant), so `runFastConfirmationRules` now accumulates the set of reasons decided across all rules and reports `didReorg` / `didRestart` from it. Closes #9545 --------- Co-authored-by: Nazar Hussain <nazarhussain@gmail.com>
1 parent e4ada6b commit 848b014

7 files changed

Lines changed: 191 additions & 23 deletions

File tree

‎dashboards/lodestar_fork_choice.json‎

Lines changed: 55 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1540,7 +1540,7 @@
15401540
},
15411541
"disableTextWrap": false,
15421542
"editorMode": "builder",
1543-
"expr": "lodestar_fast_confirmation_confirmed_slot",
1543+
"expr": "beacon_fast_confirmation_slot",
15441544
"fullMetaSearch": false,
15451545
"includeNullMetadata": true,
15461546
"instant": false,
@@ -1618,7 +1618,7 @@
16181618
"type": "prometheus",
16191619
"uid": "${DS_PROMETHEUS}"
16201620
},
1621-
"description": "",
1621+
"description": "Fast confirmation events: resets (any cause), reorgs (confirmed block became non-canonical), fallbacks to finality, and restarts from the observed unrealized justified checkpoint",
16221622
"fieldConfig": {
16231623
"defaults": {
16241624
"color": {
@@ -1661,12 +1661,12 @@
16611661
"overrides": []
16621662
},
16631663
"gridPos": {
1664-
"h": 5,
1664+
"h": 8,
16651665
"w": 12,
16661666
"x": 12,
16671667
"y": 50
16681668
},
1669-
"id": 598,
1669+
"id": 599,
16701670
"options": {
16711671
"legend": {
16721672
"calcs": [],
@@ -1688,17 +1688,65 @@
16881688
},
16891689
"disableTextWrap": false,
16901690
"editorMode": "builder",
1691-
"expr": "lodestar_fast_confirmation_resets_total",
1691+
"expr": "increase(lodestar_fast_confirmation_resets_total[$rate_interval])",
16921692
"fullMetaSearch": false,
16931693
"includeNullMetadata": true,
16941694
"instant": false,
1695-
"legendFormat": "Reset Total",
1695+
"legendFormat": "Resets",
16961696
"range": true,
16971697
"refId": "A",
16981698
"useBackend": false
1699+
},
1700+
{
1701+
"datasource": {
1702+
"type": "prometheus",
1703+
"uid": "${DS_PROMETHEUS}"
1704+
},
1705+
"disableTextWrap": false,
1706+
"editorMode": "builder",
1707+
"expr": "increase(beacon_fast_confirmation_reorgs_total[$rate_interval])",
1708+
"fullMetaSearch": false,
1709+
"includeNullMetadata": true,
1710+
"instant": false,
1711+
"legendFormat": "Reorgs",
1712+
"range": true,
1713+
"refId": "B",
1714+
"useBackend": false
1715+
},
1716+
{
1717+
"datasource": {
1718+
"type": "prometheus",
1719+
"uid": "${DS_PROMETHEUS}"
1720+
},
1721+
"disableTextWrap": false,
1722+
"editorMode": "builder",
1723+
"expr": "increase(beacon_fast_confirmation_fallbacks_total[$rate_interval])",
1724+
"fullMetaSearch": false,
1725+
"includeNullMetadata": true,
1726+
"instant": false,
1727+
"legendFormat": "Fallbacks",
1728+
"range": true,
1729+
"refId": "C",
1730+
"useBackend": false
1731+
},
1732+
{
1733+
"datasource": {
1734+
"type": "prometheus",
1735+
"uid": "${DS_PROMETHEUS}"
1736+
},
1737+
"disableTextWrap": false,
1738+
"editorMode": "builder",
1739+
"expr": "increase(beacon_fast_confirmation_restarts_total[$rate_interval])",
1740+
"fullMetaSearch": false,
1741+
"includeNullMetadata": true,
1742+
"instant": false,
1743+
"legendFormat": "Restarts",
1744+
"range": true,
1745+
"refId": "D",
1746+
"useBackend": false
16991747
}
17001748
],
1701-
"title": "Reset Total",
1749+
"title": "Fast Confirmation Events",
17021750
"type": "timeseries"
17031751
}
17041752
],

‎packages/fork-choice/src/forkChoice/fastConfirmation/fastConfirmationRule.ts‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {runFastConfirmationRules} from "./rules.ts";
77
import {
88
FastConfirmationContext,
99
FastConfirmationResult,
10+
FastConfirmationRunResult,
1011
IFastConfirmationRule,
1112
IFastConfirmationStore,
1213
} from "./types.ts";
@@ -61,7 +62,7 @@ export class FastConfirmationRule implements IFastConfirmationRule {
6162
observedJustifiedEpoch: snapshot.observedJustified.epoch,
6263
});
6364

64-
const {confirmedRoot, didReset, reason} = withObservedDuration(
65+
const {confirmedRoot, didReset, reason, didReorg, didFallback, didRestart} = withObservedDuration(
6566
this.metrics?.fastConfirmation.stepsDuration.startTimer({step: FastConfirmationSteps.runRules}),
6667
() => runFastConfirmationRules(snapshot, ctx, this.store, cache, this.logger)
6768
);
@@ -91,7 +92,7 @@ export class FastConfirmationRule implements IFastConfirmationRule {
9192
}
9293

9394
this.store.confirmedRoot = confirmedRoot;
94-
this.updateFastConfirmationMetrics(ctx, {confirmedRoot, didReset});
95+
this.updateFastConfirmationMetrics(ctx, {confirmedRoot, didReset, didReorg, didFallback, didRestart});
9596

9697
return {confirmedRoot, didReset};
9798
}
@@ -144,16 +145,28 @@ export class FastConfirmationRule implements IFastConfirmationRule {
144145
}
145146
}
146147

147-
private updateFastConfirmationMetrics(ctx: FastConfirmationContext, result: FastConfirmationResult): void {
148+
private updateFastConfirmationMetrics(
149+
ctx: FastConfirmationContext,
150+
result: Omit<FastConfirmationRunResult, "reason">
151+
): void {
148152
if (!this.metrics) return;
149153
const confirmedBlock = ctx.getBlock(result.confirmedRoot);
150154
if (confirmedBlock) {
151-
this.metrics.fastConfirmation.confirmedSlot.set(confirmedBlock.slot);
152155
this.metrics.fastConfirmation.confirmedEpoch.set(computeEpochAtSlot(confirmedBlock.slot));
156+
this.metrics.fastConfirmation.slot.set(confirmedBlock.slot);
153157
}
154158
if (result.didReset) {
155159
this.metrics.fastConfirmation.resets.inc();
156160
}
161+
if (result.didReorg) {
162+
this.metrics.fastConfirmation.reorgs.inc();
163+
}
164+
if (result.didFallback) {
165+
this.metrics.fastConfirmation.fallbacks.inc();
166+
}
167+
if (result.didRestart) {
168+
this.metrics.fastConfirmation.restarts.inc();
169+
}
157170
this.metrics.fastConfirmation.votesTracked.set(ctx.getTrackedVotesCount());
158171
}
159172
}

‎packages/fork-choice/src/forkChoice/fastConfirmation/metrics.ts‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -27,17 +27,30 @@ export function getFastConfirmationMetrics(register: MetricsRegisterExtra) {
2727
name: "lodestar_fast_confirmation_confirmed_epoch",
2828
help: "Current confirmed epoch from fast confirmation",
2929
}),
30-
confirmedSlot: register.gauge({
31-
name: "lodestar_fast_confirmation_confirmed_slot",
32-
help: "Current confirmed slot from fast confirmation",
33-
}),
3430
votesTracked: register.gauge({
3531
name: "lodestar_fast_confirmation_votes_tracked",
3632
help: "Number of checkpoint votes tracked by fast confirmation",
3733
}),
38-
resets: register.gauge({
34+
resets: register.counter({
3935
name: "lodestar_fast_confirmation_resets_total",
40-
help: "Count of fast confirmation resets due to reorgs",
36+
help: "Total number of fast confirmation resets (any cause)",
37+
}),
38+
// Standardized Fast Confirmation metrics
39+
slot: register.gauge({
40+
name: "beacon_fast_confirmation_slot",
41+
help: "Slot of the most recent confirmed block",
42+
}),
43+
reorgs: register.counter({
44+
name: "beacon_fast_confirmation_reorgs_total",
45+
help: "Total number of confirmed block reorganizations",
46+
}),
47+
fallbacks: register.counter({
48+
name: "beacon_fast_confirmation_fallbacks_total",
49+
help: "Total number of fallbacks to finality",
50+
}),
51+
restarts: register.counter({
52+
name: "beacon_fast_confirmation_restarts_total",
53+
help: "Total number of restarts from a safe unrealized justified block",
4154
}),
4255
},
4356
};

‎packages/fork-choice/src/forkChoice/fastConfirmation/rules.ts‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
FastConfirmationDecision,
88
FastConfirmationDecisionReason,
99
FastConfirmationRule,
10+
FastConfirmationRunResult,
1011
FastConfirmationSnapshot,
1112
IFastConfirmationStore,
1213
} from "./types.ts";
@@ -110,15 +111,36 @@ export function runFastConfirmationRules(
110111
store: IFastConfirmationStore,
111112
cache: FastConfirmationCache,
112113
logger?: Logger
113-
): FastConfirmationDecision {
114+
): FastConfirmationRunResult {
114115
let decision: FastConfirmationDecision = {
115116
confirmedRoot: snapshot.confirmedRoot,
116117
didReset: false,
117118
reason: FastConfirmationDecisionReason.Unchanged,
118119
};
119120

121+
// Track every reason a rule decided on, the final `decision.reason` is overwritten by
122+
// later rules (eg. a restart is followed by advancing to the latest confirmed descendant)
123+
const reasons = new Set<FastConfirmationDecisionReason>();
120124
for (const rule of FAST_CONFIRMATION_RULES) {
121125
decision = rule(snapshot, ctx, store, cache, decision, logger);
126+
reasons.add(decision.reason);
122127
}
123-
return decision;
128+
129+
// Detect a reorg directly from ancestry instead of the reset reason: when the confirmed block is
130+
// both epoch-behind and not an ancestor of head, `resetIfBehindOrNotAncestorOrUnsafe` records
131+
// `ResetBehind` (it takes precedence), so keying off `ResetNotAncestor` alone would miss reorgs
132+
// that cross an epoch boundary.
133+
const initialConfirmedBlock = getBlock(ctx, cache, snapshot.confirmedRoot);
134+
const didReorg = initialConfirmedBlock !== null && !isAncestor(ctx, cache, snapshot.headRoot, snapshot.confirmedRoot);
135+
136+
return {
137+
confirmedRoot: decision.confirmedRoot,
138+
didReset: decision.didReset,
139+
reason: decision.reason,
140+
didReorg,
141+
// A fallback is a revert to finality: a reset whose final confirmed root is the finalized
142+
// checkpoint. A later rule may advance the confirmed root forward, which is not a fallback.
143+
didFallback: decision.didReset && decision.confirmedRoot === snapshot.finalizedRoot,
144+
didRestart: reasons.has(FastConfirmationDecisionReason.ObservedJustified),
145+
};
124146
}

‎packages/fork-choice/src/forkChoice/fastConfirmation/types.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,15 @@ export type FastConfirmationDecision = {
6464
reason: FastConfirmationDecisionReason;
6565
};
6666

67+
export type FastConfirmationRunResult = FastConfirmationDecision & {
68+
/** Confirmed block became non-canonical (no longer an ancestor of head) */
69+
didReorg: boolean;
70+
/** Confirmed block reverted to the finalized checkpoint */
71+
didFallback: boolean;
72+
/** Restarted confirmation from the observed unrealized justified checkpoint */
73+
didRestart: boolean;
74+
};
75+
6776
export type FastConfirmationRule = (
6877
snapshot: FastConfirmationSnapshot,
6978
ctx: FastConfirmationContext,

‎packages/fork-choice/test/unit/forkChoice/fastConfirmation.test.ts‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,9 @@ describe("fast confirmation", () => {
268268

269269
expect(result.confirmedRoot).toBe(ZERO_ROOT);
270270
expect(result.didReset).toBe(true);
271+
expect(result.didReorg).toBe(false);
272+
expect(result.didFallback).toBe(true);
273+
expect(result.didRestart).toBe(false);
271274
});
272275

273276
it("runFastConfirmationRules advances to observed justified at epoch start", () => {
@@ -304,6 +307,11 @@ describe("fast confirmation", () => {
304307

305308
expect(result.confirmedRoot).toBe(observed.blockRoot);
306309
expect(result.didReset).toBe(true);
310+
expect(result.didReorg).toBe(false);
311+
// Confirmation advanced forward to the observed justified checkpoint, which is a restart
312+
// and not a fallback to finality even though a reset happened earlier in the pipeline
313+
expect(result.didFallback).toBe(false);
314+
expect(result.didRestart).toBe(true);
307315
});
308316

309317
it("findLatestConfirmedDescendant falls back to loop 2 when previousSlotHead is on a sibling branch", () => {
@@ -423,6 +431,59 @@ describe("fast confirmation", () => {
423431

424432
expect(result.confirmedRoot).toBe(ZERO_ROOT);
425433
expect(result.didReset).toBe(true);
434+
// Confirmed block is no longer an ancestor of head, the later descendant rule overwrites
435+
// `reason` but `didReorg` must still be reported from the reset that happened earlier
436+
expect(result.didReorg).toBe(true);
437+
expect(result.didFallback).toBe(true);
438+
expect(result.didRestart).toBe(false);
439+
});
440+
441+
it("runFastConfirmationRules reports a reorg when the confirmed block is both epoch-behind and not an ancestor of head", () => {
442+
const confirmed = makeBlock(SLOTS_PER_EPOCH - 1, ZERO_ROOT);
443+
const head = makeBlock(2 * SLOTS_PER_EPOCH, rootFromNumber(999));
444+
const blocks = [makeBlock(0, ZERO_ROOT, {blockRoot: ZERO_ROOT}), confirmed, head];
445+
const state = makeState(32, 32, [head.slot]);
446+
const store = makeStore(
447+
confirmed.blockRoot,
448+
ZERO_ROOT,
449+
ZERO_ROOT,
450+
0,
451+
0,
452+
confirmed.blockRoot,
453+
head.blockRoot,
454+
state
455+
);
456+
const ctx = makeContext(
457+
(2 * SLOTS_PER_EPOCH) as Slot,
458+
head.blockRoot,
459+
blocks,
460+
new Map(),
461+
{epoch: 0, rootHex: ZERO_ROOT},
462+
state
463+
);
464+
const snapshot = makeSnapshot(
465+
(2 * SLOTS_PER_EPOCH) as Slot,
466+
2,
467+
head.blockRoot,
468+
confirmed.blockRoot,
469+
confirmed.slot,
470+
0,
471+
ZERO_ROOT,
472+
ZERO_ROOT,
473+
0,
474+
ZERO_ROOT,
475+
0
476+
);
477+
478+
const result = runFastConfirmationRules(snapshot, ctx, store, createFastConfirmationCache());
479+
480+
expect(result.confirmedRoot).toBe(ZERO_ROOT);
481+
expect(result.didReset).toBe(true);
482+
// `resetIfBehindOrNotAncestorOrUnsafe` records `ResetBehind` (it takes precedence over
483+
// `ResetNotAncestor`), but the reorg must still be detected from ancestry
484+
expect(result.didReorg).toBe(true);
485+
expect(result.didFallback).toBe(true);
486+
expect(result.didRestart).toBe(false);
426487
});
427488

428489
it("runFastConfirmationRules only resets an unsafe confirmed chain at epoch start", () => {

‎scripts/lint-grafana-dashboard.mjs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ function assertPanels(panels) {
334334
target.expr = target.expr.replace(/job="beacon"/g, 'job=~"$beacon_job|beacon"');
335335
target.expr = target.expr.replace(/job="validator"/g, 'job=~"$validator_job|validator"');
336336

337-
// Ban use of delta and increase functions.
337+
// Ban use of the delta function.
338338
// Mixed use of delta / increase and rate make dashboards more difficult to reason about.
339339
// - delta shows the value difference based on the selected time interval, which is variable
340340
// so if time interval is X or 2*X the displayed values double.
@@ -345,9 +345,11 @@ function assertPanels(panels) {
345345
if (target.expr.includes("delta(")) {
346346
throw Error(`promql function 'delta' is not allowed, use 'rate' instead: ${target.expr}`);
347347
}
348-
if (target.expr.includes("increase(")) {
349-
throw Error(`promql function 'increase' is not allowed, use 'rate' instead: ${target.expr}`);
350-
}
348+
// increase() is allowed for rare-event counters (eg. fast confirmation reorgs/fallbacks/restarts)
349+
// where a per-window event count reads clearer than the near-zero per-second value rate would show.
350+
// if (target.expr.includes("increase(")) {
351+
// throw Error(`promql function 'increase' is not allowed, use 'rate' instead: ${target.expr}`);
352+
// }
351353
}
352354
}
353355
}

0 commit comments

Comments
 (0)