Skip to content

Commit 6d8b41b

Browse files
LandRoverLandRover
committed
feat(stream-recognizer): add New Relic metrics and NR dashboard
Emit ServerMetric events for station ticks, FFmpeg capture, decode/fingerprint steps, and Shazam provider calls so latency and success match the API pattern.
1 parent 4f0af37 commit 6d8b41b

5 files changed

Lines changed: 465 additions & 21 deletions

File tree

‎stream-recognizer/scripts/nr_dashboard.json‎

Lines changed: 283 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,283 @@
1+
{
2+
"name": "Stream Recognizer – Operational Dashboard",
3+
"permissions": "PUBLIC_READ_WRITE",
4+
"pages": [
5+
{
6+
"name": "Station ticks",
7+
"widgets": [
8+
{
9+
"title": "Tick duration P95 (ms)",
10+
"layout": { "column": 1, "row": 1, "width": 8, "height": 3 },
11+
"visualization": { "id": "viz.line" },
12+
"rawConfiguration": {
13+
"facet": { "showOtherSeries": false },
14+
"legend": { "enabled": true },
15+
"nrqlQueries": [
16+
{
17+
"accountId": 0,
18+
"query": "SELECT percentile(durationMs, 95) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerTick' FACET station TIMESERIES AUTO LIMIT 50"
19+
}
20+
],
21+
"platformOptions": { "ignoreTimeRange": false }
22+
}
23+
},
24+
{
25+
"title": "Tick error rate (%)",
26+
"layout": { "column": 9, "row": 1, "width": 4, "height": 3 },
27+
"visualization": { "id": "viz.billboard" },
28+
"rawConfiguration": {
29+
"nrqlQueries": [
30+
{
31+
"accountId": 0,
32+
"query": "SELECT percentage(count(*), WHERE success = 0) AS 'Error %' FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerTick' SINCE 1 hour ago"
33+
}
34+
],
35+
"platformOptions": { "ignoreTimeRange": false }
36+
}
37+
},
38+
{
39+
"title": "Tick outcomes (count)",
40+
"layout": { "column": 1, "row": 4, "width": 6, "height": 3 },
41+
"visualization": { "id": "viz.area" },
42+
"rawConfiguration": {
43+
"facet": { "showOtherSeries": false },
44+
"legend": { "enabled": true },
45+
"nrqlQueries": [
46+
{
47+
"accountId": 0,
48+
"query": "SELECT count(*) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerTick' FACET outcome TIMESERIES AUTO LIMIT 30"
49+
}
50+
],
51+
"platformOptions": { "ignoreTimeRange": false }
52+
}
53+
},
54+
{
55+
"title": "Outcomes by station (1h)",
56+
"layout": { "column": 7, "row": 4, "width": 6, "height": 3 },
57+
"visualization": { "id": "viz.bar" },
58+
"rawConfiguration": {
59+
"facet": { "showOtherSeries": false },
60+
"nrqlQueries": [
61+
{
62+
"accountId": 0,
63+
"query": "SELECT count(*) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerTick' FACET station, outcome SINCE 1 hour ago LIMIT 100"
64+
}
65+
],
66+
"platformOptions": { "ignoreTimeRange": false }
67+
}
68+
},
69+
{
70+
"title": "Avg tick duration by outcome (ms)",
71+
"layout": { "column": 1, "row": 7, "width": 12, "height": 3 },
72+
"visualization": { "id": "viz.bar" },
73+
"rawConfiguration": {
74+
"facet": { "showOtherSeries": false },
75+
"nrqlQueries": [
76+
{
77+
"accountId": 0,
78+
"query": "SELECT average(durationMs) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerTick' FACET outcome SINCE 6 hours ago LIMIT 30"
79+
}
80+
],
81+
"platformOptions": { "ignoreTimeRange": false }
82+
}
83+
}
84+
]
85+
},
86+
{
87+
"name": "Capture & fingerprint",
88+
"widgets": [
89+
{
90+
"title": "FFmpeg capture latency P95 (ms)",
91+
"layout": { "column": 1, "row": 1, "width": 8, "height": 3 },
92+
"visualization": { "id": "viz.line" },
93+
"rawConfiguration": {
94+
"facet": { "showOtherSeries": false },
95+
"legend": { "enabled": true },
96+
"nrqlQueries": [
97+
{
98+
"accountId": 0,
99+
"query": "SELECT percentile(durationMs, 95) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerCapture' AND success = 1 FACET station TIMESERIES AUTO LIMIT 50"
100+
}
101+
],
102+
"platformOptions": { "ignoreTimeRange": false }
103+
}
104+
},
105+
{
106+
"title": "Capture failure rate (%)",
107+
"layout": { "column": 9, "row": 1, "width": 4, "height": 3 },
108+
"visualization": { "id": "viz.bar" },
109+
"rawConfiguration": {
110+
"facet": { "showOtherSeries": false },
111+
"nrqlQueries": [
112+
{
113+
"accountId": 0,
114+
"query": "SELECT percentage(count(*), WHERE success = 0) AS 'Fail %' FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerCapture' FACET station SINCE 1 hour ago LIMIT 50"
115+
}
116+
],
117+
"platformOptions": { "ignoreTimeRange": false }
118+
}
119+
},
120+
{
121+
"title": "Capture retry depth (avg attempts)",
122+
"layout": { "column": 1, "row": 4, "width": 6, "height": 3 },
123+
"visualization": { "id": "viz.line" },
124+
"rawConfiguration": {
125+
"facet": { "showOtherSeries": false },
126+
"legend": { "enabled": true },
127+
"nrqlQueries": [
128+
{
129+
"accountId": 0,
130+
"query": "SELECT average(attempts) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerCapture' AND success = 1 FACET station TIMESERIES AUTO LIMIT 50"
131+
}
132+
],
133+
"platformOptions": { "ignoreTimeRange": false }
134+
}
135+
},
136+
{
137+
"title": "Decode (PCM) latency P95 (ms)",
138+
"layout": { "column": 7, "row": 4, "width": 6, "height": 3 },
139+
"visualization": { "id": "viz.line" },
140+
"rawConfiguration": {
141+
"facet": { "showOtherSeries": false },
142+
"legend": { "enabled": true },
143+
"nrqlQueries": [
144+
{
145+
"accountId": 0,
146+
"query": "SELECT percentile(durationMs, 95) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerDecode' AND success = 1 FACET station TIMESERIES AUTO LIMIT 50"
147+
}
148+
],
149+
"platformOptions": { "ignoreTimeRange": false }
150+
}
151+
},
152+
{
153+
"title": "Chromaprint fingerprint latency P95 (ms)",
154+
"layout": { "column": 1, "row": 7, "width": 6, "height": 3 },
155+
"visualization": { "id": "viz.line" },
156+
"rawConfiguration": {
157+
"facet": { "showOtherSeries": false },
158+
"legend": { "enabled": true },
159+
"nrqlQueries": [
160+
{
161+
"accountId": 0,
162+
"query": "SELECT percentile(durationMs, 95) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerFingerprint' AND success = 1 FACET station TIMESERIES AUTO LIMIT 50"
163+
}
164+
],
165+
"platformOptions": { "ignoreTimeRange": false }
166+
}
167+
},
168+
{
169+
"title": "Decode / fingerprint failures (count)",
170+
"layout": { "column": 7, "row": 7, "width": 6, "height": 3 },
171+
"visualization": { "id": "viz.area" },
172+
"rawConfiguration": {
173+
"facet": { "showOtherSeries": false },
174+
"legend": { "enabled": true },
175+
"nrqlQueries": [
176+
{
177+
"accountId": 0,
178+
"query": "SELECT count(*) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement IN ('StreamRecognizerDecode','StreamRecognizerFingerprint') AND success = 0 FACET measurement TIMESERIES AUTO"
179+
}
180+
],
181+
"platformOptions": { "ignoreTimeRange": false }
182+
}
183+
}
184+
]
185+
},
186+
{
187+
"name": "Providers",
188+
"widgets": [
189+
{
190+
"title": "Provider call latency P95 (ms)",
191+
"layout": { "column": 1, "row": 1, "width": 8, "height": 3 },
192+
"visualization": { "id": "viz.line" },
193+
"rawConfiguration": {
194+
"facet": { "showOtherSeries": false },
195+
"legend": { "enabled": true },
196+
"nrqlQueries": [
197+
{
198+
"accountId": 0,
199+
"query": "SELECT percentile(durationMs, 95) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerProviderCall' FACET provider TIMESERIES AUTO LIMIT 20"
200+
}
201+
],
202+
"platformOptions": { "ignoreTimeRange": false }
203+
}
204+
},
205+
{
206+
"title": "Match rate (%)",
207+
"layout": { "column": 9, "row": 1, "width": 4, "height": 3 },
208+
"visualization": { "id": "viz.billboard" },
209+
"rawConfiguration": {
210+
"nrqlQueries": [
211+
{
212+
"accountId": 0,
213+
"query": "SELECT percentage(count(*), WHERE matched = 1) AS 'Match %' FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerProviderCall' SINCE 1 hour ago"
214+
}
215+
],
216+
"platformOptions": { "ignoreTimeRange": false }
217+
}
218+
},
219+
{
220+
"title": "Provider call volume",
221+
"layout": { "column": 1, "row": 4, "width": 4, "height": 3 },
222+
"visualization": { "id": "viz.area" },
223+
"rawConfiguration": {
224+
"facet": { "showOtherSeries": false },
225+
"legend": { "enabled": true },
226+
"nrqlQueries": [
227+
{
228+
"accountId": 0,
229+
"query": "SELECT count(*) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerProviderCall' FACET provider TIMESERIES AUTO LIMIT 20"
230+
}
231+
],
232+
"platformOptions": { "ignoreTimeRange": false }
233+
}
234+
},
235+
{
236+
"title": "Transport / local failure rate (%)",
237+
"layout": { "column": 5, "row": 4, "width": 4, "height": 3 },
238+
"visualization": { "id": "viz.bar" },
239+
"rawConfiguration": {
240+
"facet": { "showOtherSeries": false },
241+
"nrqlQueries": [
242+
{
243+
"accountId": 0,
244+
"query": "SELECT percentage(count(*), WHERE success = 0) AS 'Fail %' FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerProviderCall' FACET provider SINCE 6 hours ago LIMIT 20"
245+
}
246+
],
247+
"platformOptions": { "ignoreTimeRange": false }
248+
}
249+
},
250+
{
251+
"title": "Shazam: outcomes by reason (1h)",
252+
"layout": { "column": 9, "row": 4, "width": 4, "height": 3 },
253+
"visualization": { "id": "viz.pie" },
254+
"rawConfiguration": {
255+
"facet": { "showOtherSeries": false },
256+
"nrqlQueries": [
257+
{
258+
"accountId": 0,
259+
"query": "SELECT count(*) FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerProviderCall' AND provider = 'shazam' FACET reason SINCE 1 hour ago LIMIT 20"
260+
}
261+
],
262+
"platformOptions": { "ignoreTimeRange": false }
263+
}
264+
},
265+
{
266+
"title": "Match rate by station (%)",
267+
"layout": { "column": 1, "row": 7, "width": 12, "height": 3 },
268+
"visualization": { "id": "viz.bar" },
269+
"rawConfiguration": {
270+
"facet": { "showOtherSeries": false },
271+
"nrqlQueries": [
272+
{
273+
"accountId": 0,
274+
"query": "SELECT percentage(count(*), WHERE matched = 1) AS 'Match %' FROM ServerMetric WHERE appName = 'stream-recognizer' AND measurement = 'StreamRecognizerProviderCall' AND provider = 'shazam' FACET station SINCE 6 hours ago LIMIT 50"
275+
}
276+
],
277+
"platformOptions": { "ignoreTimeRange": false }
278+
}
279+
}
280+
]
281+
}
282+
]
283+
}

‎stream-recognizer/src/index.js‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Sentry, sentryEnabled } from './sentry.js';
22
import { createHttpServer } from './http/server.js';
3-
import newrelic from 'newrelic';
43
import logger from './logger.js';
4+
import metrics from './lib/metrics.js';
55
import { initRedisStore } from './lib/redis_store.js';
66
import {
77
loadStations,
@@ -13,6 +13,8 @@ import {
1313
import { runStationTick } from './pipeline/orchestrator.js';
1414
import { probeBinary, missingBinaryHint } from './lib/binaries.js';
1515

16+
metrics.init(logger);
17+
1618
const prefix = process.env.REDIS_KEY_PREFIX || 'stream-recognizer:v2';
1719
const store = initRedisStore(logger, prefix);
1820

@@ -36,17 +38,10 @@ function onListen() {
3638
listenHost ? { port, host: listenHost } : { port },
3739
'stream-recognizer HTTP listening',
3840
);
39-
if (process.env.NEW_RELIC_ENABLED !== 'false' && process.env.NEW_RELIC_LICENSE_KEY) {
40-
try {
41-
newrelic.recordCustomEvent('ServerMetric', {
42-
measurement: 'stream_recognizer_started',
43-
started: 1,
44-
port,
45-
});
46-
} catch {
47-
/* ignore */
48-
}
49-
}
41+
metrics.report('StreamRecognizerStarted', [
42+
{ key: 'started', value: 1 },
43+
{ key: 'port', value: port },
44+
]);
5045
if (anyStationEnabled) {
5146
if (!probeBinary(ffmpegBin)) {
5247
logger.warn(

‎stream-recognizer/src/lib/metrics.js‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
import newrelic from 'newrelic';
2+
import { hostname } from 'node:os';
3+
4+
/**
5+
* Custom metrics as `ServerMetric` events (same shape as server) for New Relic dashboards.
6+
*/
7+
class MetricsWrapper {
8+
/** @type {import('pino').Logger | null} */
9+
logger = null;
10+
11+
/** @param {import('pino').Logger} Logger */
12+
init(Logger) {
13+
this.logger = Logger;
14+
}
15+
16+
_enabled() {
17+
return process.env.NEW_RELIC_ENABLED !== 'false' && Boolean(process.env.NEW_RELIC_LICENSE_KEY);
18+
}
19+
20+
/**
21+
* @param {string} measurementId
22+
* @param {{ key: string; value: string | number }[]} fields
23+
*/
24+
report(measurementId, fields = []) {
25+
if (!this._enabled()) {
26+
return this;
27+
}
28+
try {
29+
const attrs = {
30+
measurement: measurementId,
31+
host: hostname(),
32+
};
33+
for (const f of fields) {
34+
attrs[f.key] = f.value;
35+
}
36+
newrelic.recordCustomEvent('ServerMetric', attrs);
37+
} catch (err) {
38+
this.logger?.warn?.({
39+
message: 'New Relic custom metric failed',
40+
measurementId,
41+
err,
42+
});
43+
}
44+
return this;
45+
}
46+
47+
/**
48+
* @param {string} measurementId
49+
* @param {() => Promise<any>} fn
50+
* @param {Record<string, string | number>} [attrs]
51+
*/
52+
async timeIt(measurementId, fn, attrs = {}) {
53+
const start = Date.now();
54+
let success = 1;
55+
try {
56+
return await fn();
57+
} catch (err) {
58+
success = 0;
59+
throw err;
60+
} finally {
61+
this.report(measurementId, [
62+
{ key: 'durationMs', value: Date.now() - start },
63+
{ key: 'success', value: success },
64+
...Object.entries(attrs).map(([k, v]) => ({ key: k, value: v })),
65+
]);
66+
}
67+
}
68+
}
69+
70+
export default new MetricsWrapper();

0 commit comments

Comments
 (0)