make it not download 27mb on page load

Author: sekuba

README.md

@@ -45,7 +45,7 @@ The backend API is rooted at `/api`. `/live` reports process liveness and
 
 | Method | Route | Purpose |
 | --- | --- | --- |
-| `GET` | `/api/config`, `/api/status`, `/api/network` | Capabilities, freshness, protocol state, and public cases |
+| `GET` | `/api/config`, `/api/status`, `/api/network` | Capabilities; freshness and protocol state; public cases (ETag-revalidatable) |
 | `GET` | `/api/sequencers/:address`, `/api/cases/:id` | Public sequencer and exact-case views |
 | `POST` | `/api/watches` | Create a private watch and return its management token once |
 | `GET/PATCH/DELETE` | `/api/watches/:id` | Bearer-authenticated watch management |

collector/src/case-api-server.mjs

@@ -1,5 +1,6 @@
 import http from 'node:http';
-import { randomUUID } from 'node:crypto';
+import { createHash, randomUUID } from 'node:crypto';
+import { gzipSync } from 'node:zlib';
 
 import {
   createOpaqueToken,
@@ -89,7 +90,7 @@ export class CaseApiServer {
         } else {
           this.logger?.debug?.('API request rejected', details);
         }
-        this.sendError(response, error);
+        this.sendError(request, response, error);
       });
     });
   }
@@ -125,14 +126,14 @@ export class CaseApiServer {
     }
 
     if (request.method === 'GET' && url.pathname === '/live') {
-      return this.send(response, 200, { status: 'live' });
+      return this.send(request, response, 200, { status: 'live' });
     }
     if (request.method === 'GET' && url.pathname === '/health') {
       const status = this.status();
-      return this.send(response, status.status === 'healthy' ? 200 : 503, status);
+      return this.send(request, response, status.status === 'healthy' ? 200 : 503, status);
     }
     if (request.method === 'GET' && url.pathname === `${API_PREFIX}/config`) {
-      return this.send(response, 200, {
+      return this.send(request, response, 200, {
         network: this.network,
         maxSequencers: this.maxSequencers,
         notifications: {
@@ -147,29 +148,35 @@ export class CaseApiServer {
       });
     }
     if (request.method === 'GET' && url.pathname === `${API_PREFIX}/status`) {
-      return this.send(response, 200, this.status());
+      return this.send(request, response, 200, this.status());
     }
     if (request.method === 'GET' && url.pathname === `${API_PREFIX}/network`) {
-      return this.send(response, 200, {
-        ...this.repository.getNetworkSummary(this.network),
-        sources: this.status().sources,
-      });
+      return this.send(
+        request,
+        response,
+        200,
+        this.repository.getNetworkSummary(this.network),
+        { revalidate: true },
+      );
     }
 
     const sequencerMatch = /^\/api\/sequencers\/(0x[0-9a-fA-F]{40})$/.exec(
       url.pathname,
     );
     if (request.method === 'GET' && sequencerMatch) {
-      return this.send(response, 200, this.repository.getSequencerRecord(
-        sequencerMatch[1],
-        this.network,
-      ));
+      return this.send(
+        request,
+        response,
+        200,
+        this.repository.getSequencerRecord(sequencerMatch[1], this.network),
+        { revalidate: true },
+      );
     }
     const caseMatch = /^\/api\/cases\/([^/]+)$/.exec(url.pathname);
     if (request.method === 'GET' && caseMatch) {
       const item = this.repository.getCase(decodeURIComponent(caseMatch[1]));
       if (!item) throw new InputError('case_not_found', 'Slashing case not found', 404);
-      return this.send(response, 200, item);
+      return this.send(request, response, 200, item, { revalidate: true });
     }
 
     if (request.method === 'POST' && url.pathname === `${API_PREFIX}/watches`) {
@@ -186,7 +193,7 @@ export class CaseApiServer {
         addresses,
         now: this.now(),
       });
-      return this.send(response, 201, {
+      return this.send(request, response, 201, {
         watch: publicWatch(watch, this.repository),
         managementToken,
       });
@@ -196,7 +203,7 @@ export class CaseApiServer {
     if (watchMatch) {
       const watch = this.authorizeWatch(request, watchMatch[1]);
       if (request.method === 'GET') {
-        return this.send(response, 200, publicWatch(watch, this.repository));
+        return this.send(request, response, 200, publicWatch(watch, this.repository));
       }
       this.limitMutation(request);
       if (request.method === 'PATCH') {
@@ -208,7 +215,7 @@ export class CaseApiServer {
           addresses,
           now: this.now(),
         });
-        return this.send(response, 200, publicWatch(updated, this.repository));
+        return this.send(request, response, 200, publicWatch(updated, this.repository));
       }
       if (request.method === 'DELETE') {
         this.repository.deleteWatch(watch.id);
@@ -242,7 +249,7 @@ export class CaseApiServer {
           configJson: JSON.stringify(subscription),
           now: this.now(),
         });
-        return this.send(response, 200, publicWatch(updated, this.repository));
+        return this.send(request, response, 200, publicWatch(updated, this.repository));
       }
       if (request.method === 'DELETE') {
         this.repository.deleteEndpoint(watch.id, 'web_push');
@@ -274,7 +281,7 @@ export class CaseApiServer {
         expiresAt,
         now: this.now(),
       });
-      return this.send(response, 201, {
+      return this.send(request, response, 201, {
         url: `https://t.me/${this.telegramBotUsername}?start=${token}`,
         expiresAt: new Date(expiresAt).toISOString(),
       });
@@ -295,7 +302,7 @@ export class CaseApiServer {
           409,
         );
       }
-      return this.send(response, 202, { queued });
+      return this.send(request, response, 202, { queued });
     }
 
     throw new InputError('not_found', 'Route not found', 404);
@@ -421,25 +428,46 @@ export class CaseApiServer {
     response.setHeader('access-control-allow-origin', this.corsOrigin);
     response.setHeader('access-control-allow-methods', 'GET,POST,PATCH,PUT,DELETE,OPTIONS');
     response.setHeader('access-control-allow-headers', 'authorization,content-type');
-    response.setHeader('vary', 'Origin');
+    response.setHeader('vary', 'Origin, Accept-Encoding');
   }
 
-  send(response, status, value) {
+  // Public data endpoints send `cache-control: no-cache` plus a weak ETag so
+  // browsers revalidate every poll and receive a bodyless 304 while nothing
+  // changed. Private and mutating responses stay `no-store`. Bodies are
+  // gzipped at the origin: the network path to the CDN edge is metered.
+  send(request, response, status, value, { revalidate = false } = {}) {
     const body = JSON.stringify(value);
-    response.writeHead(status, {
+    const headers = {
       'content-type': 'application/json; charset=utf-8',
-      'cache-control': 'no-store',
-      'content-length': Buffer.byteLength(body),
-    });
-    response.end(body);
+      'cache-control': revalidate ? 'no-cache' : 'no-store',
+    };
+    if (revalidate && status === 200) {
+      const etag = `W/"${createHash('sha256').update(body).digest('base64url')}"`;
+      headers.etag = etag;
+      const ifNoneMatch = request.headers['if-none-match'];
+      if (typeof ifNoneMatch === 'string' && ifNoneMatch.includes(etag)) {
+        response.writeHead(304, headers);
+        response.end();
+        return;
+      }
+    }
+    const acceptsGzip = /(?:^|[,\s])gzip(?:$|[;,])/
+      .test(String(request.headers['accept-encoding'] ?? ''));
+    const payload = acceptsGzip && Buffer.byteLength(body) > 1_024
+      ? gzipSync(body)
+      : body;
+    if (payload !== body) headers['content-encoding'] = 'gzip';
+    headers['content-length'] = Buffer.byteLength(payload);
+    response.writeHead(status, headers);
+    response.end(payload);
   }
 
-  sendError(response, error) {
+  sendError(request, response, error) {
     const safeStatus = errorStatus(error);
     if (error?.retryAfterMs) {
       response.setHeader('retry-after', String(Math.ceil(error.retryAfterMs / 1_000)));
     }
-    this.send(response, safeStatus, {
+    this.send(request, response, safeStatus, {
       error: {
         code: error?.code ?? 'internal_error',
         message: safeStatus === 500

collector/src/case-repository.mjs

@@ -63,12 +63,50 @@ export class CaseRepository {
       this.db.exec('PRAGMA foreign_keys = ON');
       this.db.exec('PRAGMA busy_timeout = 5000');
       this.initializeSchema();
+      this.pruneResult = this.pruneSupersededRoundObservations();
     } catch (error) {
       this.db.close();
       throw error;
     }
   }
 
+  // Databases written before superseded l1_round rows were deleted on
+  // reconcile still carry every historical vote-state progression. Remove the
+  // rows whose canonical replacement for the same round exists in the same
+  // case, and rebuild the affected case projections. Idempotent: after the
+  // first run there is nothing left to match.
+  pruneSupersededRoundObservations() {
+    return this.transaction(() => {
+      const rows = this.db.prepare(`
+        SELECT id, observation_json AS observationJson
+        FROM observations AS superseded
+        WHERE source = 'ethereum_l1' AND kind = 'l1_round' AND canonical = 0
+          AND COALESCE(
+            json_extract(observation_json, '$.data.historicalExecution'), 0
+          ) != 1
+          AND EXISTS (
+            SELECT 1 FROM observations AS replacement
+            WHERE replacement.canonical = 1
+              AND replacement.kind = 'l1_round'
+              AND replacement.network = superseded.network
+              AND replacement.lineage_id = superseded.lineage_id
+              AND replacement.sequencer = superseded.sequencer
+              AND replacement.target_epoch = superseded.target_epoch
+              AND replacement.round = superseded.round
+          )
+      `).all();
+      const affected = new Set();
+      for (const row of rows) {
+        this.db.prepare('DELETE FROM observations WHERE id = ?').run(row.id);
+        affected.add(caseIdFor(parseJson(row.observationJson, null)));
+      }
+      const projection = affected.size > 0
+        ? this.reprojectCases([...affected], { notify: false })
+        : { changed: 0 };
+      return { pruned: rows.length, casesChanged: projection.changed };
+    });
+  }
+
   initializeSchema() {
     const applicationId = Number(
       this.db.prepare('PRAGMA application_id').get().application_id,
@@ -493,6 +531,11 @@ export class CaseRepository {
         SELECT case_json AS caseJson FROM cases WHERE id = ?
       `).get(caseId);
       const previous = previousRow ? parseJson(previousRow.caseJson, null) : null;
+      // Deleting a superseded round row can remove a case's earliest
+      // observation; the moment the case was first seen must survive that.
+      if (previous && previous.firstObservedAt < current.firstObservedAt) {
+        current.firstObservedAt = previous.firstObservedAt;
+      }
       const currentJson = JSON.stringify(current);
       if (previousRow?.caseJson === currentJson) continue;
       this.db.prepare(`
@@ -590,10 +633,12 @@ export class CaseRepository {
     };
   }
 
+  // The network feed deliberately omits the protocol snapshot and source
+  // health: both change every poll and live in /api/status, while this
+  // response only changes when a case does — which keeps its ETag stable.
   getNetworkSummary(selectedNetwork) {
     const cases = this.listCases({ network: selectedNetwork });
     return {
-      protocol: this.getProtocolSnapshot(),
       summary: summarizeNetwork(cases),
       cases,
     };
@@ -989,28 +1034,45 @@ export class CaseRepository {
     });
   }
 
+  // A fresh snapshot supersedes the stored vote-state row of every covered
+  // round it re-reports with different data. Superseded rows are deleted:
+  // they are poll-cadence progressions of the same round, not independent
+  // chain evidence, and retaining them made cases grow without bound. A row
+  // whose target vanished from a covered round is different — that evidence
+  // was removed on L1, so it is kept as a non-canonical correction.
+  // Historical execution rows come from Slashed logs and are only ever
+  // invalidated by the reorg path, never replaced by a newer snapshot.
   reconcileL1RoundObservations(snapshot, current) {
     const seen = new Set(current.map((item) => item.id));
+    const replaced = new Set(current.map((item) =>
+      `${item.lineageId}:${item.round}:${item.sequencer}:${item.targetEpoch}`));
     const coverage = new Set((snapshot.stacks ?? []).flatMap((stack) =>
       (stack.rounds ?? []).map((round) =>
         `${address(stack.proposerAddress, 'SlashingProposer')}:${unsignedString(round.round, 'round')}`)));
     if (coverage.size === 0) return [];
     const rows = this.db.prepare(`
-      SELECT id, lineage_id AS lineageId, round, observation_json AS observationJson
+      SELECT id, lineage_id AS lineageId, round, sequencer,
+        target_epoch AS targetEpoch, observation_json AS observationJson
       FROM observations
       WHERE source = 'ethereum_l1' AND kind = 'l1_round' AND canonical = 1
     `).all();
     const affected = new Set();
     for (const row of rows) {
       if (!coverage.has(`${row.lineageId}:${row.round}`) || seen.has(row.id)) continue;
       const observation = parseJson(row.observationJson, null);
-      observation.provenance.canonical = false;
-      observation.provenance.invalidatedAt = toIso(
-        snapshot.blockTimestamp ?? snapshot.observedAt,
-      );
-      this.db.prepare(`
-        UPDATE observations SET canonical = 0, observation_json = ? WHERE id = ?
-      `).run(JSON.stringify(observation), row.id);
+      if (observation?.data?.historicalExecution) continue;
+      const key = `${row.lineageId}:${row.round}:${row.sequencer}:${row.targetEpoch}`;
+      if (replaced.has(key)) {
+        this.db.prepare('DELETE FROM observations WHERE id = ?').run(row.id);
+      } else {
+        observation.provenance.canonical = false;
+        observation.provenance.invalidatedAt = toIso(
+          snapshot.blockTimestamp ?? snapshot.observedAt,
+        );
+        this.db.prepare(`
+          UPDATE observations SET canonical = 0, observation_json = ? WHERE id = ?
+        `).run(JSON.stringify(observation), row.id);
+      }
       affected.add(caseIdFor(observation));
     }
     return [...affected];

collector/src/main.mjs

@@ -20,6 +20,9 @@ async function main() {
   const config = loadConfig();
   const logger = new Logger(config.logLevel);
   const repository = new CaseRepository(config.databasePath);
+  if (repository.pruneResult?.pruned > 0) {
+    logger.info('Pruned superseded L1 round observations', repository.pruneResult);
+  }
   try {
     repository.bindRuntimeIdentity({
       network: config.network,