Skip to content

Commit bae2de1

Browse files
dtfiedlerclaude
dtfiedler
and
claude
committed
feat(monitoring): add gateway uptime monitoring and alerts
- Add gateway_monitors, gateway_healthcheck_history, gateway_monitor_webhooks tables - Add healthcheck service to check /ar-io/info endpoint - Add monitor processor with cron jobs (every minute check, daily history prune) - Add webhook and email notifications for gateway down/recovery alerts - Add API endpoints for monitor CRUD, history, and webhook linking - Support max 3 monitors per subscriber with configurable intervals and thresholds Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent f4ff0a0 commit bae2de1

9 files changed

Lines changed: 1731 additions & 2 deletions

File tree

‎src/config.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,14 @@ export const twitterApiSecret = process.env.TWITTER_API_SECRET;
9595
export const arnsResolverUrl =
9696
process.env.ARNS_RESOLVER_URL || 'https://permagate.io/ar-io/resolver';
9797
export const disableArnsSync = process.env.DISABLE_ARNS_SYNC === 'true';
98+
99+
// Gateway Monitoring
100+
export const disableGatewayMonitoring =
101+
process.env.DISABLE_GATEWAY_MONITORING === 'true';
102+
export const gatewayHealthcheckTimeoutMs = process.env
103+
.GATEWAY_HEALTHCHECK_TIMEOUT_MS
104+
? +process.env.GATEWAY_HEALTHCHECK_TIMEOUT_MS
105+
: 10000;
106+
export const maxMonitorsPerSubscriber = process.env.MAX_MONITORS_PER_SUBSCRIBER
107+
? +process.env.MAX_MONITORS_PER_SUBSCRIBER
108+
: 3;

‎src/db/migrations/20260114120000_add_gateway_monitors.ts‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
import type { Knex } from 'knex';
2+
3+
export async function up(knex: Knex): Promise<void> {
4+
// Gateway monitors table - main configuration
5+
await knex.schema.createTable('gateway_monitors', (table) => {
6+
table.increments('id').primary();
7+
table
8+
.integer('subscriber_id')
9+
.notNullable()
10+
.references('id')
11+
.inTable('subscribers')
12+
.onDelete('CASCADE');
13+
table.text('fqdn').notNullable(); // Gateway domain (e.g., ar-io.dev)
14+
table.boolean('enabled').notNullable().defaultTo(true);
15+
table.integer('check_interval_minutes').notNullable().defaultTo(5);
16+
table.integer('failure_threshold').notNullable().defaultTo(3);
17+
table.text('current_status').notNullable().defaultTo('unknown'); // 'unknown' | 'healthy' | 'unhealthy'
18+
table.integer('consecutive_failures').notNullable().defaultTo(0);
19+
table.timestamp('last_check_at');
20+
table.timestamp('last_alert_sent_at');
21+
table.timestamp('last_recovery_sent_at');
22+
table.boolean('notify_email').notNullable().defaultTo(true);
23+
table.timestamp('created_at').notNullable().defaultTo(knex.fn.now());
24+
table.timestamp('updated_at').notNullable().defaultTo(knex.fn.now());
25+
26+
// One monitor per gateway per subscriber
27+
table.unique(['subscriber_id', 'fqdn']);
28+
// Index for efficient cron queries
29+
table.index(['enabled', 'last_check_at']);
30+
});
31+
32+
// Gateway healthcheck history table - stores results (14-day retention)
33+
await knex.schema.createTable('gateway_healthcheck_history', (table) => {
34+
table.increments('id').primary();
35+
table
36+
.integer('monitor_id')
37+
.notNullable()
38+
.references('id')
39+
.inTable('gateway_monitors')
40+
.onDelete('CASCADE');
41+
table.text('status').notNullable(); // 'success' | 'failed'
42+
table.integer('response_time_ms'); // null if failed
43+
table.integer('status_code'); // HTTP status code, null if connection failed
44+
table.text('error_message'); // Error details if failed
45+
table.timestamp('checked_at').notNullable().defaultTo(knex.fn.now());
46+
47+
// Index for history queries
48+
table.index(['monitor_id', 'checked_at']);
49+
});
50+
51+
// Gateway monitor webhooks table - links monitors to notification webhooks
52+
await knex.schema.createTable('gateway_monitor_webhooks', (table) => {
53+
table.increments('id').primary();
54+
table
55+
.integer('monitor_id')
56+
.notNullable()
57+
.references('id')
58+
.inTable('gateway_monitors')
59+
.onDelete('CASCADE');
60+
table
61+
.integer('webhook_id')
62+
.notNullable()
63+
.references('id')
64+
.inTable('webhooks')
65+
.onDelete('CASCADE');
66+
table.boolean('notify_on_down').notNullable().defaultTo(true);
67+
table.boolean('notify_on_recovery').notNullable().defaultTo(true);
68+
table.timestamp('created_at').notNullable().defaultTo(knex.fn.now());
69+
70+
// One link per monitor/webhook pair
71+
table.unique(['monitor_id', 'webhook_id']);
72+
});
73+
}
74+
75+
export async function down(knex: Knex): Promise<void> {
76+
await knex.schema.dropTableIfExists('gateway_monitor_webhooks');
77+
await knex.schema.dropTableIfExists('gateway_healthcheck_history');
78+
await knex.schema.dropTableIfExists('gateway_monitors');
79+
}

‎src/db/schema.ts‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,76 @@ const arnsNameSubscriptionSchema = z.object({
308308

309309
export type ArNSNameSubscription = z.infer<typeof arnsNameSubscriptionSchema>;
310310

311+
// Gateway Monitor schemas
312+
export const gatewayMonitorStatuses = [
313+
'unknown',
314+
'healthy',
315+
'unhealthy',
316+
] as const;
317+
export type GatewayMonitorStatus = (typeof gatewayMonitorStatuses)[number];
318+
319+
const gatewayMonitorSchema = z.object({
320+
id: z.number(),
321+
subscriber_id: z.number(),
322+
fqdn: z.string(),
323+
enabled: z.boolean().default(true),
324+
check_interval_minutes: z.number().default(5),
325+
failure_threshold: z.number().default(3),
326+
current_status: z.enum(gatewayMonitorStatuses).default('unknown'),
327+
consecutive_failures: z.number().default(0),
328+
last_check_at: z.date().nullable(),
329+
last_alert_sent_at: z.date().nullable(),
330+
last_recovery_sent_at: z.date().nullable(),
331+
notify_email: z.boolean().default(true),
332+
created_at: z.date(),
333+
updated_at: z.date(),
334+
});
335+
336+
export type GatewayMonitor = z.infer<typeof gatewayMonitorSchema>;
337+
338+
const newGatewayMonitorSchema = gatewayMonitorSchema.omit({
339+
id: true,
340+
current_status: true,
341+
consecutive_failures: true,
342+
last_check_at: true,
343+
last_alert_sent_at: true,
344+
last_recovery_sent_at: true,
345+
created_at: true,
346+
updated_at: true,
347+
});
348+
349+
export type NewGatewayMonitor = z.infer<typeof newGatewayMonitorSchema>;
350+
351+
// Gateway healthcheck history schema
352+
export const healthcheckStatuses = ['success', 'failed'] as const;
353+
export type HealthcheckStatus = (typeof healthcheckStatuses)[number];
354+
355+
const gatewayHealthcheckHistorySchema = z.object({
356+
id: z.number(),
357+
monitor_id: z.number(),
358+
status: z.enum(healthcheckStatuses),
359+
response_time_ms: z.number().nullable(),
360+
status_code: z.number().nullable(),
361+
error_message: z.string().nullable(),
362+
checked_at: z.date(),
363+
});
364+
365+
export type GatewayHealthcheckHistory = z.infer<
366+
typeof gatewayHealthcheckHistorySchema
367+
>;
368+
369+
// Gateway monitor webhook link schema
370+
const gatewayMonitorWebhookSchema = z.object({
371+
id: z.number(),
372+
monitor_id: z.number(),
373+
webhook_id: z.number(),
374+
notify_on_down: z.boolean().default(true),
375+
notify_on_recovery: z.boolean().default(true),
376+
created_at: z.date(),
377+
});
378+
379+
export type GatewayMonitorWebhook = z.infer<typeof gatewayMonitorWebhookSchema>;
380+
311381
// Export schemas for validation
312382
export const schemas = {
313383
subscriber: subscriberSchema,
@@ -322,4 +392,8 @@ export const schemas = {
322392
newArnsName: newArnsNameSchema,
323393
arnsExpirationNotification: arnsExpirationNotificationSchema,
324394
arnsNameSubscription: arnsNameSubscriptionSchema,
395+
gatewayMonitor: gatewayMonitorSchema,
396+
newGatewayMonitor: newGatewayMonitorSchema,
397+
gatewayHealthcheckHistory: gatewayHealthcheckHistorySchema,
398+
gatewayMonitorWebhook: gatewayMonitorWebhookSchema,
325399
};

0 commit comments

Comments
 (0)