diff --git a/.env.example b/.env.example index ff1c1e0..10568f1 100644 --- a/.env.example +++ b/.env.example @@ -22,15 +22,6 @@ NEXT_PUBLIC_SEARCH_API_BASE= STUDIO_API_BASE= STUDIO_TOKEN= -# Live Studio shares. LIVE_REFRESH_PRINCIPAL is the credential dfo-apis and -# dfo-ai ask of a card refresh. It has the same value on all three services -# and, when deployed, is mounted from Secret Manager by dfo-infra. Empty -# means the Share dialog offers no Live Data and every Studio share is a -# snapshot. LIVE_SHARE_WINDOW_S is how long a refreshed card's answer stands -# before the next viewer asks again (default 300, at most 900). -LIVE_REFRESH_PRINCIPAL= -# LIVE_SHARE_WINDOW_S=300 - # Accounts and saved conversations (Postgres). Local development only -- # run `just db-up && just db-migrate` first, then this connects as the # `website_runtime` role db/migrate-local.ts creates. Deployed services @@ -121,10 +112,17 @@ SENTRY_AUTH_TOKEN= # attached by the proxy so outside MCP clients don't need it. Unset: the # caller's own Authorization header is passed through. MCP_API_TOKEN= -# /mcp names each caller to dfo-ai as x-dfo-caller: ip:, an -# HMAC-SHA-256 of the client IP keyed by this secret (e.g. openssl rand -hex -# 32). If unset, no caller is sent and the site logs one warning. +# /mcp names each caller to dfo-ai, and the dfo-apis proxy to dfo-apis, as +# x-dfo-caller: ip:, an HMAC-SHA-256 of the client address (an IPv6 +# address as its /64) keyed by this secret (e.g. openssl rand -hex 32). It +# must equal dfo-apis's DFO_CALLER_HMAC_KEY. If unset, no caller is sent and +# the site logs one warning. CALLER_HMAC_KEY= +# The secret dfo-apis checks before it trusts x-dfo-caller (its +# DFO_CALLER_PRINCIPAL), sent as x-dfo-caller-principal on every dfo-apis +# call. If unset, the site sends dfo-apis neither header and logs one warning, +# and dfo-apis counts the whole site as one caller. +DFO_CALLER_PRINCIPAL= # Which X-Forwarded-For entry is the client, counted from the right. 0 (the # default) is the rightmost, the one Cloud Run's front end appends; add one # for each load balancer in front of the service. diff --git a/db/auth-schema.ts b/db/auth-schema.ts index fc80610..9e70410 100644 --- a/db/auth-schema.ts +++ b/db/auth-schema.ts @@ -12,6 +12,9 @@ export const user = pgTable('user', { .defaultNow() .$onUpdate(() => /* @__PURE__ */ new Date()) .notNull(), + // Set when the account's deletion begins (`user.additionalFields` in + // src/lib/auth/auth.ts), so no share token is stored for it after. + deletingAt: timestamp('deleting_at', { withTimezone: true }), }); export const session = pgTable( diff --git a/db/migrations/0022_share_token.sql b/db/migrations/0022_share_token.sql new file mode 100644 index 0000000..d860326 --- /dev/null +++ b/db/migrations/0022_share_token.sql @@ -0,0 +1,8 @@ +CREATE TABLE "share_token" ( + "share_id" text PRIMARY KEY NOT NULL, + "user_id" text NOT NULL, + "write_token" text NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "share_token" ADD CONSTRAINT "share_token_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action; \ No newline at end of file diff --git a/db/migrations/0023_share_cleanup_and_deleting_at.sql b/db/migrations/0023_share_cleanup_and_deleting_at.sql new file mode 100644 index 0000000..1e87a20 --- /dev/null +++ b/db/migrations/0023_share_cleanup_and_deleting_at.sql @@ -0,0 +1,9 @@ +CREATE TABLE "share_cleanup" ( + "share_id" text PRIMARY KEY NOT NULL, + "user_id" text NOT NULL, + "write_token" text NOT NULL, + "reason" text NOT NULL, + "recorded_at" timestamp with time zone DEFAULT now() NOT NULL +); +--> statement-breakpoint +ALTER TABLE "user" ADD COLUMN "deleting_at" timestamp with time zone; \ No newline at end of file diff --git a/db/migrations/meta/0022_snapshot.json b/db/migrations/meta/0022_snapshot.json new file mode 100644 index 0000000..9e0b19f --- /dev/null +++ b/db/migrations/meta/0022_snapshot.json @@ -0,0 +1,1144 @@ +{ + "id": "a27f5d07-abf1-471c-92b4-36943717ea39", + "prevId": "3a669c64-ea49-434a-9f75-2f090520da67", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.conversation": { + "name": "conversation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_session_id": { + "name": "agent_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "turn_count": { + "name": "turn_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "conversation_user_updated_idx": { + "name": "conversation_user_updated_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "conversation_user_id_user_id_fk": { + "name": "conversation_user_id_user_id_fk", + "tableFrom": "conversation", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.event": { + "name": "event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "event_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visitor_id": { + "name": "visitor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "props": { + "name": "props", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": { + "event_created_at_idx": { + "name": "event_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "event_name_created_at_idx": { + "name": "event_name_created_at_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.feedback": { + "name": "feedback", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "feedback_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "feedback_created_at_idx": { + "name": "feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "feedback_message_length": { + "name": "feedback_message_length", + "value": "char_length(\"feedback\".\"message\") BETWEEN 1 AND 5000" + } + }, + "isRLSEnabled": false + }, + "public.limit_counter": { + "name": "limit_counter", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "window_start": { + "name": "window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "limit_counter_window_start_idx": { + "name": "limit_counter_window_start_idx", + "columns": [ + { + "expression": "window_start", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.roadmap_vote": { + "name": "roadmap_vote", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entry_id": { + "name": "entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "roadmap_vote_entry_idx": { + "name": "roadmap_vote_entry_idx", + "columns": [ + { + "expression": "entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "roadmap_vote_user_id_user_id_fk": { + "name": "roadmap_vote_user_id_user_id_fk", + "tableFrom": "roadmap_vote", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "roadmap_vote_user_id_entry_id_pk": { + "name": "roadmap_vote_user_id_entry_id_pk", + "columns": [ + "user_id", + "entry_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share": { + "name": "share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "card_id": { + "name": "card_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "share_user_idx": { + "name": "share_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_user_id_user_id_fk": { + "name": "share_user_id_user_id_fk", + "tableFrom": "share", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_card_result": { + "name": "share_card_result", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "card_id": { + "name": "card_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "card": { + "name": "card", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "retryable": { + "name": "retryable", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "retry_after_s": { + "name": "retry_after_s", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "fresh_until": { + "name": "fresh_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "share_card_result_share_id_share_id_fk": { + "name": "share_card_result_share_id_share_id_fk", + "tableFrom": "share_card_result", + "tableTo": "share", + "columnsFrom": [ + "share_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "share_card_result_share_id_card_id_pk": { + "name": "share_card_result_share_id_card_id_pk", + "columns": [ + "share_id", + "card_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_ref": { + "name": "share_ref", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "part_index": { + "name": "part_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity": { + "name": "identity", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entity_groups": { + "name": "entity_groups", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "variable_text": { + "name": "variable_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_version": { + "name": "content_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "artifact_digest": { + "name": "artifact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "assumptions": { + "name": "assumptions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "share_ref_share_id_share_id_fk": { + "name": "share_ref_share_id_share_id_fk", + "tableFrom": "share_ref", + "tableTo": "share", + "columnsFrom": [ + "share_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "share_ref_share_id_part_index_pk": { + "name": "share_ref_share_id_part_index_pk", + "columns": [ + "share_id", + "part_index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_token": { + "name": "share_token", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "write_token": { + "name": "write_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "share_token_user_id_user_id_fk": { + "name": "share_token_user_id_user_id_fk", + "tableFrom": "share_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webmail_domain": { + "name": "webmail_domain", + "schema": "", + "columns": { + "domain": { + "name": "domain", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/db/migrations/meta/0023_snapshot.json b/db/migrations/meta/0023_snapshot.json new file mode 100644 index 0000000..141c929 --- /dev/null +++ b/db/migrations/meta/0023_snapshot.json @@ -0,0 +1,1194 @@ +{ + "id": "5c9656fc-9b6b-44f8-9c1e-a3ca5c5aaeeb", + "prevId": "a27f5d07-abf1-471c-92b4-36943717ea39", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.conversation": { + "name": "conversation", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "agent_session_id": { + "name": "agent_session_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "turn_count": { + "name": "turn_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "conversation_user_updated_idx": { + "name": "conversation_user_updated_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "updated_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "conversation_user_id_user_id_fk": { + "name": "conversation_user_id_user_id_fk", + "tableFrom": "conversation", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.event": { + "name": "event", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "event_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "visitor_id": { + "name": "visitor_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "props": { + "name": "props", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + } + }, + "indexes": { + "event_created_at_idx": { + "name": "event_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + }, + "event_name_created_at_idx": { + "name": "event_name_created_at_idx", + "columns": [ + { + "expression": "name", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.feedback": { + "name": "feedback", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigint", + "primaryKey": true, + "notNull": true, + "identity": { + "type": "always", + "name": "feedback_id_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "page": { + "name": "page", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + } + }, + "indexes": { + "feedback_created_at_idx": { + "name": "feedback_created_at_idx", + "columns": [ + { + "expression": "created_at", + "isExpression": false, + "asc": false, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": { + "feedback_message_length": { + "name": "feedback_message_length", + "value": "char_length(\"feedback\".\"message\") BETWEEN 1 AND 5000" + } + }, + "isRLSEnabled": false + }, + "public.limit_counter": { + "name": "limit_counter", + "schema": "", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "window_start": { + "name": "window_start", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "limit_counter_window_start_idx": { + "name": "limit_counter_window_start_idx", + "columns": [ + { + "expression": "window_start", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.roadmap_vote": { + "name": "roadmap_vote", + "schema": "", + "columns": { + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "entry_id": { + "name": "entry_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "roadmap_vote_entry_idx": { + "name": "roadmap_vote_entry_idx", + "columns": [ + { + "expression": "entry_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "roadmap_vote_user_id_user_id_fk": { + "name": "roadmap_vote_user_id_user_id_fk", + "tableFrom": "roadmap_vote", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "roadmap_vote_user_id_entry_id_pk": { + "name": "roadmap_vote_user_id_entry_id_pk", + "columns": [ + "user_id", + "entry_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share": { + "name": "share", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "mode": { + "name": "mode", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "conversation_id": { + "name": "conversation_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "card_id": { + "name": "card_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "share_user_idx": { + "name": "share_user_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "share_user_id_user_id_fk": { + "name": "share_user_id_user_id_fk", + "tableFrom": "share", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_card_result": { + "name": "share_card_result", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "card_id": { + "name": "card_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "card": { + "name": "card", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "frame": { + "name": "frame", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "code": { + "name": "code", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "retryable": { + "name": "retryable", + "type": "boolean", + "primaryKey": false, + "notNull": false + }, + "retry_after_s": { + "name": "retry_after_s", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "checked_at": { + "name": "checked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "fresh_until": { + "name": "fresh_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "lease_until": { + "name": "lease_until", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "share_card_result_share_id_share_id_fk": { + "name": "share_card_result_share_id_share_id_fk", + "tableFrom": "share_card_result", + "tableTo": "share", + "columnsFrom": [ + "share_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "share_card_result_share_id_card_id_pk": { + "name": "share_card_result_share_id_card_id_pk", + "columns": [ + "share_id", + "card_id" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_cleanup": { + "name": "share_cleanup", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "write_token": { + "name": "write_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "recorded_at": { + "name": "recorded_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_ref": { + "name": "share_ref", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "part_index": { + "name": "part_index", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "spec_id": { + "name": "spec_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "identity": { + "name": "identity", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "entity_groups": { + "name": "entity_groups", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "variable_text": { + "name": "variable_text", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "content_version": { + "name": "content_version", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "artifact_digest": { + "name": "artifact_digest", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "operations": { + "name": "operations", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "assumptions": { + "name": "assumptions", + "type": "jsonb", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": { + "share_ref_share_id_share_id_fk": { + "name": "share_ref_share_id_share_id_fk", + "tableFrom": "share_ref", + "tableTo": "share", + "columnsFrom": [ + "share_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "share_ref_share_id_part_index_pk": { + "name": "share_ref_share_id_part_index_pk", + "columns": [ + "share_id", + "part_index" + ] + } + }, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.share_token": { + "name": "share_token", + "schema": "", + "columns": { + "share_id": { + "name": "share_id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "write_token": { + "name": "write_token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "share_token_user_id_user_id_fk": { + "name": "share_token_user_id_user_id_fk", + "tableFrom": "share_token", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.webmail_domain": { + "name": "webmail_domain", + "schema": "", + "columns": { + "domain": { + "name": "domain", + "type": "text", + "primaryKey": true, + "notNull": true + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.account": { + "name": "account", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "account_userId_idx": { + "name": "account_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.session": { + "name": "session", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "session_userId_idx": { + "name": "session_userId_idx", + "columns": [ + { + "expression": "user_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "session_token_unique": { + "name": "session_token_unique", + "nullsNotDistinct": false, + "columns": [ + "token" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.user": { + "name": "user", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "email_verified": { + "name": "email_verified", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "deleting_at": { + "name": "deleting_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "user_email_unique": { + "name": "user_email_unique", + "nullsNotDistinct": false, + "columns": [ + "email" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.verification": { + "name": "verification", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "expires_at": { + "name": "expires_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + { + "expression": "identifier", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/db/migrations/meta/_journal.json b/db/migrations/meta/_journal.json index 31bc3a0..0499520 100644 --- a/db/migrations/meta/_journal.json +++ b/db/migrations/meta/_journal.json @@ -148,6 +148,20 @@ "when": 1792794347981, "tag": "0021_account_daily_before_today", "breakpoints": true + }, + { + "idx": 22, + "version": "7", + "when": 1793196112763, + "tag": "0022_share_token", + "breakpoints": true + }, + { + "idx": 23, + "version": "7", + "when": 1793258249005, + "tag": "0023_share_cleanup_and_deleting_at", + "breakpoints": true } ] } \ No newline at end of file diff --git a/db/schema.ts b/db/schema.ts index a773bba..6b8969b 100644 --- a/db/schema.ts +++ b/db/schema.ts @@ -151,6 +151,40 @@ export const shareCardResult = pgTable( export type ShareCardResultRow = typeof shareCardResult.$inferSelect; +/** + * The write token dfo-apis issued for a share a signed-in user made, which + * the website sends on that user's replace and delete requests + * (src/lib/chat/shareTokens.ts). The share itself lives in dfo-apis, so + * `share_id` references no table here. A guest's share has no row and + * stays as made. + */ +export const shareToken = pgTable('share_token', { + shareId: text('share_id').primaryKey(), + userId: text('user_id') + .notNull() + .references(() => user.id, { onDelete: 'cascade' }), + writeToken: text('write_token').notNull(), + createdAt: timestamp('created_at', { withTimezone: true }).defaultNow().notNull(), +}); + +export type ShareTokenRow = typeof shareToken.$inferSelect; + +/** + * A share in dfo-apis that no stored write token could take down: its + * token was refused when the owner's account was deleted, or the take-down + * that follows a token the website could not keep failed. `user_id` + * references no table, so the row outlives the account. An operator takes + * each share down with dfo-apis' `datasets-maintenance take-down-share + * ` and deletes the row. `reason` is the code the take-down failed with. + */ +export const shareCleanup = pgTable('share_cleanup', { + shareId: text('share_id').primaryKey(), + userId: text('user_id').notNull(), + writeToken: text('write_token').notNull(), + reason: text('reason').notNull(), + recordedAt: timestamp('recorded_at', { withTimezone: true }).defaultNow().notNull(), +}); + /** * One row per (user, roadmap entry) vote: the "Under consideration" items * on /data/roadmap, which only signed-in people can vote for. The entry id diff --git a/db/share-export.ts b/db/share-export.ts new file mode 100644 index 0000000..3750881 --- /dev/null +++ b/db/share-export.ts @@ -0,0 +1,246 @@ +import type { Pool, PoolClient } from 'pg'; + +import { asRecord } from '../src/lib/asRecord.ts'; +import type { SavedConversation } from '../src/lib/chat/history.ts'; +import type { ShareMode } from '../src/lib/chat/shareMode.ts'; +import { slotCanvas, viewerCopy, type ShareSlot } from '../src/lib/chat/sharePayload.ts'; +import type { ShareKind } from '../src/lib/chat/sharePrune.ts'; + +/** + * The share export (scripts/export-shares.ts): every row of `share` made at + * or after a moment, as the JSON lines docs/share-export.md defines, which + * dfo-apis' `import-shares` reads. `placeShare` decides where each stored + * share lands, and refuses a row it cannot place with the reason. + * `share_card_result` rows are a cache of live refreshes and are not read. + */ + +/** One data card of a share as dfo-apis holds it, in display order. */ +export type ExportSlot = ShareSlot; + +/** One line of the export. */ +export type ExportLine = { + id: string; + kind: ShareKind; + mode: ShareMode; + owner: string | null; + created_at: string; + title: string; + slots: ExportSlot[]; + payload: Record; +}; + +/** One `share_ref` row, under its column names. */ +export type StoredPart = { + part_index: number; + spec_id: string; + identity: unknown; + entity_groups: unknown; + variable_text: string | null; + content_version: string | null; + artifact_digest: string | null; + operations: unknown; + assumptions: unknown; +}; + +/** One `share` row with its `share_ref` rows in part order. `created_at` + * is the stored moment in UTC to the microsecond. */ +export type StoredShare = { + id: string; + kind: string; + mode: string | null; + user_id: string | null; + created_at: string; + title: string; + data: unknown; + parts: StoredPart[]; +}; + +export type Placement = { line: ExportLine } | { refusal: string }; + +type LineBase = Pick; + +const SHARE_ID = /^[A-Za-z0-9]{8}$/; + +const isCanvasState = (data: unknown): data is SavedConversation => { + const state = asRecord(data); + + return ( + Array.isArray(state?.['widgets']) && + Array.isArray(state['pinnedIds']) && + Array.isArray(state['removedIds']) + ); +}; + +const placeSearch = (row: StoredShare, base: LineBase): Placement => { + if (row.mode !== null && row.mode !== 'live' && row.mode !== 'snapshot') { + return { refusal: `mode ${JSON.stringify(row.mode)} is not live or snapshot` }; + } + + // A search share with no stored mode is live (searchShareMode). + const mode: ShareMode = row.mode ?? 'live'; + const unpinned = row.parts.find((part) => part.content_version === null); + + if (mode === 'snapshot' && unpinned) { + return { refusal: `snapshot part ${unpinned.part_index} has no content_version` }; + } + + const stored = asRecord(row.data)?.['query']; + const query = typeof stored === 'string' ? stored : null; + + return { + line: { + ...base, + kind: 'search', + mode, + slots: [], + payload: { query: query ?? row.title, parts: row.parts }, + }, + }; +}; + +const placeStudio = ( + row: StoredShare, + kind: 'conversation' | 'canvas' | 'card', + base: LineBase, +): Placement => { + if (row.parts.length > 0) { + return { refusal: `a ${kind} share has ${row.parts.length} share_ref rows` }; + } + + if (row.mode !== null && row.mode !== 'live' && row.mode !== 'snapshot') { + return { refusal: `mode ${JSON.stringify(row.mode)} is not live or snapshot` }; + } + + // A Studio share with no stored mode is a snapshot (isLiveStudioShare). + const mode: ShareMode = row.mode ?? 'snapshot'; + + if (kind === 'conversation') { + if (mode === 'live') { + return { refusal: 'a conversation share cannot be live' }; + } + + const data = asRecord(row.data); + + if (!data) { + return { refusal: 'data is not a saved conversation' }; + } + + return { line: { ...base, kind, mode, slots: [], payload: viewerCopy(data) } }; + } + + if (!isCanvasState(row.data)) { + return { refusal: 'data has no widgets, pinnedIds and removedIds arrays' }; + } + + const slotted = slotCanvas(row.data, mode); + + if ('unslotted' in slotted) { + return { + refusal: `live card ${slotted.unslotted[0]} has no component ref in dfo-apis' form`, + }; + } + + const { slots, widgets } = slotted; + + return { + line: { ...base, kind, mode, slots, payload: { ...viewerCopy(row.data), widgets } }, + }; +}; + +/** Where one stored share lands in the export, or why it cannot. */ +export const placeShare = (row: StoredShare): Placement => { + if (!SHARE_ID.test(row.id)) { + return { refusal: 'the id is not 8 letters and digits' }; + } + + const base: LineBase = { + id: row.id, + owner: row.user_id, + created_at: row.created_at, + title: row.title, + }; + + switch (row.kind) { + case 'search': + return placeSearch(row, base); + case 'conversation': + case 'canvas': + case 'card': + return placeStudio(row, row.kind, base); + default: + return { refusal: `kind ${JSON.stringify(row.kind)} is not a share kind` }; + } +}; + +/** + * Every share made at or after `since` (an ISO timestamp with its offset), + * by `created_at` then id, each with its `share_ref` rows in part order. + * Both reads run in one repeatable-read transaction, so they see the same + * snapshot and a share changed between them still exports whole. + */ +export const readShares = async (pool: Pool, since: string): Promise => { + const client = await pool.connect(); + + try { + await client.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY'); + + const rows = await readSharesIn(client, since); + + await client.query('COMMIT'); + + return rows; + } catch (error) { + await client.query('ROLLBACK').catch(() => {}); + + throw error; + } finally { + client.release(); + } +}; + +const readSharesIn = async (client: PoolClient, since: string): Promise => { + const shares = await client.query>( + `SELECT id, kind, mode, user_id, title, data, + to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"') AS created_at + FROM share + WHERE created_at >= $1::timestamptz + ORDER BY share.created_at, id`, + [since], + ); + const parts = await client.query( + `SELECT r.share_id, r.part_index, r.spec_id, r.identity, r.entity_groups, + r.variable_text, r.content_version, r.artifact_digest, r.operations, + r.assumptions + FROM share_ref r JOIN share s ON s.id = r.share_id + WHERE s.created_at >= $1::timestamptz + ORDER BY r.share_id, r.part_index`, + [since], + ); + const partsOf = new Map(); + + for (const { share_id: shareId, ...part } of parts.rows) { + partsOf.set(shareId, [...(partsOf.get(shareId) ?? []), part]); + } + + return shares.rows.map((row) => ({ ...row, parts: partsOf.get(row.id) ?? [] })); +}; + +/** The export of `rows`: one JSON line each, or every refusal by share id. */ +export const exportShares = ( + rows: StoredShare[], +): { lines: ExportLine[]; refusals: { id: string; reason: string }[] } => { + const lines: ExportLine[] = []; + const refusals: { id: string; reason: string }[] = []; + + for (const row of rows) { + const placement = placeShare(row); + + if ('line' in placement) { + lines.push(placement.line); + } else { + refusals.push({ id: row.id, reason: placement.refusal }); + } + } + + return { lines, refusals }; +}; diff --git a/db/share-token-import.ts b/db/share-token-import.ts new file mode 100644 index 0000000..687aada --- /dev/null +++ b/db/share-token-import.ts @@ -0,0 +1,105 @@ +import { inArray } from 'drizzle-orm'; +import type { NodePgDatabase } from 'drizzle-orm/node-postgres'; + +import { putShareToken, shareTokensFenced } from '../src/lib/chat/shareTokens.ts'; +import { user } from './schema.ts'; + +/** + * The token import (scripts/import-share-tokens.ts): the write tokens + * dfo-apis' `import-shares` minted for signed-in owners' shares, one JSON + * line `{id, owner, write_token}` each, into `share_token`. A line whose + * token is already stored stores it again, so a second run of the same file + * changes nothing. + */ + +export type TokenLine = { id: string; owner: string; write_token: string }; + +const SHARE_ID = /^[A-Za-z0-9]{8}$/; +const WRITE_TOKEN = /^shw_\S+$/; + +const isTokenLine = (value: unknown): value is TokenLine => { + if (typeof value !== 'object' || value === null) { + return false; + } + + const { id, owner, write_token: token } = value as Record; + + return ( + typeof id === 'string' && + SHARE_ID.test(id) && + typeof owner === 'string' && + owner !== '' && + typeof token === 'string' && + WRITE_TOKEN.test(token) + ); +}; + +/** The lines of a token file, or the 1-based numbers of the lines that are + * not `{id, owner, write_token}` with an 8-character id and a `shw_` token. + * Blank lines are skipped. */ +export const readTokenLines = (text: string): { lines: TokenLine[]; invalid: number[] } => { + const lines: TokenLine[] = []; + const invalid: number[] = []; + + text.split('\n').forEach((raw, index) => { + if (raw.trim() === '') { + return; + } + + let parsed: unknown; + + try { + parsed = JSON.parse(raw); + } catch { + parsed = null; + } + + if (isTokenLine(parsed)) { + lines.push(parsed); + } else { + invalid.push(index + 1); + } + }); + + return { lines, invalid }; +}; + +/** What an import came to: how many tokens it stored, the shares whose + * owner is no longer a website user, the shares whose owner's account is + * being deleted, which store no token, and the shares whose token another + * user holds, which are left as they were. */ +export type TokenImport = { + stored: number; + ownerGone: string[]; + ownerDeleting: string[]; + heldByOther: string[]; +}; + +export const importShareTokens = async ( + db: Pick, + lines: TokenLine[], +): Promise => { + const owners = [...new Set(lines.map((line) => line.owner))]; + const present = owners.length + ? new Set( + (await db.select({ id: user.id }).from(user).where(inArray(user.id, owners))).map( + ({ id }) => id, + ), + ) + : new Set(); + const result: TokenImport = { stored: 0, ownerGone: [], ownerDeleting: [], heldByOther: [] }; + + for (const { id, owner, write_token: writeToken } of lines) { + if (!present.has(owner)) { + result.ownerGone.push(id); + } else if (await putShareToken(db, { shareId: id, userId: owner, writeToken })) { + result.stored += 1; + } else if (await shareTokensFenced(db, owner)) { + result.ownerDeleting.push(id); + } else { + result.heldByOther.push(id); + } + } + + return result; +}; diff --git a/db/test/apis-shares.test.ts b/db/test/apis-shares.test.ts new file mode 100644 index 0000000..f0f63b9 --- /dev/null +++ b/db/test/apis-shares.test.ts @@ -0,0 +1,560 @@ +import path from 'node:path'; +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { drizzle } from 'drizzle-orm/node-postgres'; + +import { + createOwnedShare, + ownerShareWrite, + takeDownUserShares, + type SharesBackend, +} from '../../src/lib/chat/apisShares.ts'; +import { putShareToken, readShareToken } from '../../src/lib/chat/shareTokens.ts'; +import { applyMigrations } from '../apply-migrations.ts'; +import { user } from '../schema.ts'; +import { createTestDatabase } from './harness.ts'; + +const MIGRATIONS_FOLDER = path.resolve(import.meta.dirname, '../migrations'); + +type Call = { url: string; method: string; token: string | null }; + +// A dfo-apis that answers each call with `answer`, recording the method, +// path, and the write token it was sent. +const backendOf = (answer: (call: Call) => Response | Promise) => { + const calls: Call[] = []; + const backend: SharesBackend = { + base: 'https://apis.example', + nameCaller: () => {}, + fetch: async (input, init) => { + const call = { + url: String(input).replace('https://apis.example', ''), + method: init?.method ?? 'GET', + token: new Headers(init?.headers).get('x-dfo-write-token'), + }; + + calls.push(call); + + return answer(call); + }, + }; + + return { backend, calls }; +}; + +const created = () => + Response.json({ id: 'Ab3dEf7h', write_token: 'shw_made', revision: 1 }, { status: 201 }); + +/** + * `createOwnedShare` and `ownerShareWrite` against the migrated tables as + * the runtime role and a stubbed dfo-apis: a signed-in maker's token is kept + * and sent back on their writes, a guest's is not kept, and a caller with no + * token, or whose token dfo-apis refuses, gets `not_found`. + */ +test('owner writes to shares held in dfo-apis', async (t) => { + const database = await createTestDatabase(); + t.after(() => database.teardown()); + + await applyMigrations(database.migratorPool(), { + migrationsFolder: MIGRATIONS_FOLDER, + runtimeDbUser: database.runtimeUser, + migratorDbUser: database.migratorUser, + }); + + const db = drizzle(database.runtimePool()); + + await db.insert(user).values([ + { id: 'owner', name: 'O', email: 'o@one.org' }, + { id: 'other', name: 'X', email: 'x@one.org' }, + ]); + + const body = { kind: 'canvas' as const, latest: true, title: 'T', cards: [], payload: {} }; + const cleanups = async () => + ( + await database + .runtimePool() + .query( + 'select share_id, user_id, write_token, reason from share_cleanup order by share_id', + ) + ).rows; + const deletingAt = async (id: string) => + ( + await database + .runtimePool() + .query<{ deleting_at: Date | null }>( + 'select deleting_at from "user" where id = $1', + [id], + ) + ).rows[0]?.deleting_at ?? null; + // Until a statement in this database waits on a lock, for at most 5 seconds. + const waitForLockWait = async () => { + const watcher = database.adminPool(); + const until = Date.now() + 5_000; + + while (Date.now() < until) { + const { rows } = await watcher.query( + `select 1 from pg_stat_activity + where datname = $1 and wait_event_type = 'Lock'`, + [database.databaseName], + ); + + if (rows.length > 0) { + return; + } + + await new Promise((resolve) => setTimeout(resolve, 10)); + } + + throw new Error('no statement waited on a lock'); + }; + + await t.test("a signed-in maker's token is kept, and a guest's is not", async () => { + const { backend } = backendOf(created); + + assert.deepEqual(await createOwnedShare({ db, backend }, 'owner', body), { + id: 'Ab3dEf7h', + }); + assert.equal( + await readShareToken(db, { shareId: 'Ab3dEf7h', userId: 'owner' }), + 'shw_made', + ); + + await database.runtimePool().query('delete from share_token'); + await createOwnedShare({ db, backend }, null, body); + assert.equal( + (await database.runtimePool().query('select count(*)::int as n from share_token')) + .rows[0].n, + 0, + ); + }); + + // dfo-apis' read of a share of `kind`, and its answer to every write. + const holding = + (kind: string, write: () => Response = () => Response.json({ id: 'Ab3dEf7h' })) => + (call: Call) => + call.method === 'GET' + ? Response.json({ + id: 'Ab3dEf7h', + kind, + mode: 'live', + title: null, + slots: [], + payload: null, + created_at: '2026-10-28T10:00:00Z', + }) + : write(); + + await t.test("a switch to snapshot sends the owner's token and keeps it", async () => { + await putShareToken(db, { shareId: 'Ab3dEf7h', userId: 'owner', writeToken: 'shw_made' }); + + const { backend, calls } = backendOf(holding('canvas')); + const write = { shareId: 'Ab3dEf7h', userId: 'owner' }; + + assert.equal( + await ownerShareWrite({ db, backend }, { ...write, action: 'snapshot' }), + 'done', + ); + assert.deepEqual(calls, [ + { url: '/api/presentation/shares/Ab3dEf7h', method: 'GET', token: null }, + { + url: '/api/presentation/shares/Ab3dEf7h/snapshot', + method: 'POST', + token: 'shw_made', + }, + ]); + assert.equal(await readShareToken(db, write), 'shw_made'); + }); + + await t.test( + 'a search or conversation share is not switched, and keeps its token', + async () => { + for (const kind of ['search', 'conversation']) { + const { backend, calls } = backendOf(holding(kind)); + const write = { shareId: 'Ab3dEf7h', userId: 'owner' }; + + assert.equal( + await ownerShareWrite({ db, backend }, { ...write, action: 'snapshot' }), + 'unsupported_kind', + kind, + ); + assert.deepEqual( + calls.map(({ method }) => method), + ['GET'], + kind, + ); + assert.equal(await readShareToken(db, write), 'shw_made', kind); + } + }, + ); + + await t.test( + 'a switch refused for a reason of its own throws its code and keeps the token', + async () => { + // A slot's frame is gone, so dfo-apis cannot pin it: the share stays. + const { backend } = backendOf( + holding('canvas', () => + Response.json({ code: 'frame_unavailable', retryable: false }, { status: 404 }), + ), + ); + const write = { shareId: 'Ab3dEf7h', userId: 'owner' }; + + await assert.rejects( + ownerShareWrite({ db, backend }, { ...write, action: 'snapshot' }), + { + name: 'SharesError', + code: 'frame_unavailable', + }, + ); + assert.equal(await readShareToken(db, write), 'shw_made'); + }, + ); + + await t.test('a caller with no token gets not_found and calls nothing', async () => { + const { backend, calls } = backendOf(() => new Response(null, { status: 204 })); + + assert.equal( + await ownerShareWrite( + { db, backend }, + { shareId: 'Ab3dEf7h', userId: 'other', action: 'delete' }, + ), + 'not_found', + ); + assert.equal(calls.length, 0); + }); + + await t.test('a delete sends the token, then drops it', async () => { + const { backend, calls } = backendOf(() => new Response(null, { status: 204 })); + const write = { shareId: 'Ab3dEf7h', userId: 'owner' }; + + assert.equal( + await ownerShareWrite({ db, backend }, { ...write, action: 'delete' }), + 'done', + ); + assert.deepEqual(calls, [ + { url: '/api/presentation/shares/Ab3dEf7h', method: 'DELETE', token: 'shw_made' }, + ]); + assert.equal(await readShareToken(db, write), null); + }); + + await t.test('a token dfo-apis refuses, or a share it no longer has, is dropped', async () => { + for (const [status, code] of [ + [403, 'write_token_invalid'], + [401, 'write_token_required'], + [404, 'not_found'], + ] as const) { + await putShareToken(db, { + shareId: 'Ab3dEf7h', + userId: 'owner', + writeToken: 'shw_old', + }); + + const { backend } = backendOf( + holding('canvas', () => Response.json({ code }, { status })), + ); + const write = { shareId: 'Ab3dEf7h', userId: 'owner' }; + + assert.equal( + await ownerShareWrite({ db, backend }, { ...write, action: 'snapshot' }), + 'not_found', + ); + assert.equal(await readShareToken(db, write), null, code); + } + }); + + await t.test('another refusal throws and keeps the token', async () => { + await putShareToken(db, { shareId: 'Ab3dEf7h', userId: 'owner', writeToken: 'shw_made' }); + + const { backend } = backendOf(() => + Response.json({ code: 'service_unavailable', retryable: true }, { status: 503 }), + ); + const write = { shareId: 'Ab3dEf7h', userId: 'owner' }; + + await assert.rejects(ownerShareWrite({ db, backend }, { ...write, action: 'delete' }), { + name: 'SharesError', + code: 'service_unavailable', + }); + assert.equal(await readShareToken(db, write), 'shw_made'); + }); + + await t.test( + 'before an account goes, each of its shares is taken down with its token, a lost token counts as gone, and a failure stops there', + async () => { + await database.runtimePool().query('delete from share_token'); + + for (const [shareId, userId] of [ + ['Aaaaaaa1', 'owner'], + ['Aaaaaaa2', 'owner'], + ['Aaaaaaa3', 'owner'], + ['Bbbbbbb1', 'other'], + ]) { + await putShareToken(db, { shareId, userId, writeToken: `shw_${shareId}` }); + } + + const tokens = async () => + ( + await database + .runtimePool() + .query<{ share_id: string }>( + 'select share_id from share_token order by share_id', + ) + ).rows.map(({ share_id: id }) => id); + + // The first share is gone already, the second's token is lost, + // which leaves nothing to try again, and dfo-apis fails the third. + const failing = backendOf(({ url }) => + url.endsWith('Aaaaaaa1') + ? Response.json({ code: 'not_found' }, { status: 404 }) + : url.endsWith('Aaaaaaa2') + ? Response.json({ code: 'write_token_invalid' }, { status: 403 }) + : url.endsWith('Aaaaaaa3') + ? Response.json( + { code: 'service_unavailable', retryable: true }, + { status: 503 }, + ) + : new Response(null, { status: 204 }), + ); + + await assert.rejects(takeDownUserShares({ db, backend: failing.backend }, 'owner'), { + name: 'SharesError', + code: 'service_unavailable', + }); + assert.deepEqual(failing.calls, [ + { + url: '/api/presentation/shares/Aaaaaaa1', + method: 'DELETE', + token: 'shw_Aaaaaaa1', + }, + { + url: '/api/presentation/shares/Aaaaaaa2', + method: 'DELETE', + token: 'shw_Aaaaaaa2', + }, + { + url: '/api/presentation/shares/Aaaaaaa3', + method: 'DELETE', + token: 'shw_Aaaaaaa3', + }, + ]); + assert.deepEqual(await tokens(), ['Aaaaaaa3', 'Bbbbbbb1']); + // The refused token is kept for an operator, and the stopped + // deletion lifts its fence, so the owner can share again. + assert.deepEqual(await cleanups(), [ + { + share_id: 'Aaaaaaa2', + user_id: 'owner', + write_token: 'shw_Aaaaaaa2', + reason: 'write_token_invalid', + }, + ]); + assert.equal(await deletingAt('owner'), null); + + const { backend, calls } = backendOf(() => new Response(null, { status: 204 })); + + await takeDownUserShares({ db, backend }, 'owner'); + assert.deepEqual( + calls.map(({ url }) => url), + ['/api/presentation/shares/Aaaaaaa3'], + ); + assert.deepEqual(await tokens(), ['Bbbbbbb1']); + assert.notEqual(await deletingAt('owner'), null); + }, + ); + + // Until `done` holds, for at most 5 seconds. + const until = async (done: () => boolean) => { + const deadline = Date.now() + 5_000; + + while (!done()) { + assert.ok(Date.now() < deadline, 'timed out'); + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }; + // A dfo-apis that holds each take-down until `open` is called, then + // answers it with `answer`. + const gated = (answer: () => Response) => { + let open = () => {}; + const opened = new Promise((resolve) => { + open = resolve; + }); + + return { ...backendOf(async () => opened.then(answer)), open }; + }; + const fenceMicros = async () => + BigInt( + ( + await database.runtimePool().query<{ us: string }>( + `select (extract(epoch from deleting_at) * 1e6)::bigint::text as us + from "user" where id = 'owner'`, + ) + ).rows[0]!.us, + ); + const reset = async () => { + await database.runtimePool().query('delete from share_token'); + await database.runtimePool().query('delete from share_cleanup'); + await database + .runtimePool() + .query(`update "user" set deleting_at = null where id = 'owner'`); + }; + + await t.test( + 'an attempt that fails leaves the fence a later attempt set, which then finishes', + async () => { + await reset(); + await putShareToken(db, { shareId: 'Aaaaaaa1', userId: 'owner', writeToken: 'shw_1' }); + + const first = gated(() => + Response.json({ code: 'service_unavailable', retryable: true }, { status: 503 }), + ); + const firstRun = takeDownUserShares({ db, backend: first.backend }, 'owner'); + + await until(() => first.calls.length === 1); + + const second = gated(() => new Response(null, { status: 204 })); + const secondRun = takeDownUserShares({ db, backend: second.backend }, 'owner'); + + await until(() => second.calls.length === 1); + + const held = await fenceMicros(); + + first.open(); + await assert.rejects(firstRun, { code: 'service_unavailable' }); + assert.equal(await fenceMicros(), held); + assert.equal( + await putShareToken(db, { + shareId: 'Aaaaaaa2', + userId: 'owner', + writeToken: 'shw_2', + }), + false, + ); + + second.open(); + await secondRun; + assert.equal(await readShareToken(db, { shareId: 'Aaaaaaa1', userId: 'owner' }), null); + }, + ); + + await t.test( + 'an attempt sets its fence afresh before each take-down, and takes down a token stored meanwhile', + async () => { + await reset(); + + for (const shareId of ['Aaaaaaa1', 'Aaaaaaa2']) { + await putShareToken(db, { shareId, userId: 'owner', writeToken: `shw_${shareId}` }); + } + + const fences: bigint[] = []; + const { backend, calls } = backendOf(async ({ url }) => { + fences.push(await fenceMicros()); + + // A token that reached the table past the fence, as one would + // once a fence lapsed. + if (url.endsWith('Aaaaaaa1')) { + await database.adminPool().query( + `insert into share_token (share_id, user_id, write_token) + values ('Aaaaaaa9', 'owner', 'shw_Aaaaaaa9')`, + ); + } + + return new Response(null, { status: 204 }); + }); + + await takeDownUserShares({ db, backend }, 'owner'); + assert.deepEqual( + calls.map(({ url, token }) => [url, token]), + [ + ['/api/presentation/shares/Aaaaaaa1', 'shw_Aaaaaaa1'], + ['/api/presentation/shares/Aaaaaaa2', 'shw_Aaaaaaa2'], + ['/api/presentation/shares/Aaaaaaa9', 'shw_Aaaaaaa9'], + ], + ); + assert.ok(fences[0]! < fences[1]! && fences[1]! < fences[2]!, String(fences)); + assert.equal( + (await database.runtimePool().query('select 1 from share_token')).rowCount, + 0, + ); + }, + ); + + await t.test('a fenced account makes no share and calls dfo-apis for none', async () => { + await reset(); + await database + .runtimePool() + .query(`update "user" set deleting_at = now() where id = 'owner'`); + + const { backend, calls } = backendOf(created); + + await assert.rejects(createOwnedShare({ db, backend }, 'owner', body), { + name: 'SharesError', + code: 'account_deleting', + retryable: false, + }); + assert.equal(calls.length, 0); + }); + + await t.test( + 'a share made once the account is fenced keeps no token and is taken down, or goes to cleanup when that fails', + async () => { + await database.runtimePool().query('delete from share_cleanup'); + await database + .runtimePool() + .query(`update "user" set deleting_at = null where id = 'owner'`); + + const answering = (takeDown: () => Response) => + backendOf((call) => (call.method === 'DELETE' ? takeDown() : created())); + + // The fence is set, uncommitted, when the token insert starts: the + // insert waits on the user row, then sees the committed fence. + const fencer = await database.adminPool().connect(); + + try { + await fencer.query('begin'); + await fencer.query(`update "user" set deleting_at = now() where id = 'owner'`); + + const { backend, calls } = answering(() => new Response(null, { status: 204 })); + const making = assert.rejects(createOwnedShare({ db, backend }, 'owner', body), { + code: 'token_not_kept', + }); + + await waitForLockWait(); + await fencer.query('commit'); + await making; + assert.deepEqual( + calls.map(({ method, token }) => [method, token]), + [ + ['POST', null], + ['DELETE', 'shw_made'], + ], + ); + } finally { + fencer.release(); + } + + const { backend } = answering(() => + Response.json({ code: 'service_unavailable', retryable: true }, { status: 503 }), + ); + + // With the fence lifted, another user's row holds the id, and the + // take-down fails. + await database + .runtimePool() + .query(`update "user" set deleting_at = null where id = 'owner'`); + await putShareToken(db, { shareId: 'Ab3dEf7h', userId: 'other', writeToken: 'shw_x' }); + await assert.rejects(createOwnedShare({ db, backend }, 'owner', body), { + code: 'token_not_kept', + }); + assert.equal(await readShareToken(db, { shareId: 'Ab3dEf7h', userId: 'owner' }), null); + assert.deepEqual(await cleanups(), [ + { + share_id: 'Ab3dEf7h', + user_id: 'owner', + write_token: 'shw_made', + reason: 'service_unavailable', + }, + ]); + + // The row outlives the account. + await database.runtimePool().query(`delete from "user" where id = 'owner'`); + assert.equal((await cleanups()).length, 1); + }, + ); +}); diff --git a/db/test/share-export.test.ts b/db/test/share-export.test.ts new file mode 100644 index 0000000..747b77c --- /dev/null +++ b/db/test/share-export.test.ts @@ -0,0 +1,525 @@ +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import assert from 'node:assert/strict'; +import test from 'node:test'; +import type { Pool, PoolClient } from 'pg'; + +import { applyMigrations } from '../apply-migrations.ts'; +import { exportShares, placeShare, readShares, type StoredShare } from '../share-export.ts'; +import { createTestDatabase } from './harness.ts'; + +const MIGRATIONS_FOLDER = path.resolve(import.meta.dirname, '../migrations'); +const SCRIPT = path.resolve(import.meta.dirname, '../../scripts/export-shares.ts'); +const run = promisify(execFile); + +const FRAME = { spec_id: 'spec_aid', content_version: 'cv_1' }; +const DERIVED = { spec_id: 'ops_growth', content_version: 'ocv_2' }; +const ref = (n: number) => ({ component_id: `cmp_card${n}`, revision: n }); + +const chart = (id: string, turn: number, extra: Record = {}) => ({ + kind: 'chart', + id, + turn, + chartType: 'line', + title: `Chart ${id}`, + categories: ['2023', '2024'], + series: [{ label: 'Kenya', values: [1, 2] }], + ...extra, +}); +const table = (id: string, turn: number, extra: Record = {}) => ({ + kind: 'table', + id, + turn, + title: `Table ${id}`, + columns: ['Year', 'Value'], + rows: [['2024', '3']], + ...extra, +}); +const note = (id: string, turn: number) => ({ kind: 'text', id, turn, title: 'Note', body: 'B' }); + +const canvas = (widgets: unknown[], rest: Record = {}) => ({ + id: 'conv-1', + sessionId: 'harness-session', + componentRefs: [ref(9)], + title: 'Aid to Kenya', + updatedAt: 1, + messages: [], + widgets, + activity: [], + turnCount: 2, + pinnedIds: [], + removedIds: [], + layout: { c1: { col: 0, row: 0, w: 4, h: 3 } }, + theme: 'paper', + textCards: [{ kind: 'text', id: 'tc1', title: 'Mine', body: 'Words' }], + titles: { c1: 'Renamed' }, + notes: { c1: 'A note' }, + views: { c1: 'table' }, + ...rest, +}); + +const SEARCH_PART = { + spec_id: 'spec_aid', + identity: { variable: 'aid' }, + entity_groups: null, + variable_text: 'aid to Kenya', + operations: [], + assumptions: null, +}; + +// One row per legacy case, inserted in this order with these moments. +const ROWS = [ + // A guest's search share with no mode: live, its parts unpinned. + { + id: 'srchLiv1', + kind: 'search', + mode: null, + user_id: null, + created_at: '2026-09-01T10:00:00.000001Z', + title: 'aid to Kenya', + data: { query: 'aid to Kenya' }, + parts: [ + { ...SEARCH_PART, part_index: 1, content_version: null, artifact_digest: null }, + { ...SEARCH_PART, part_index: 0, content_version: null, artifact_digest: null }, + ], + }, + // A search snapshot made before artifact_digest was stored. + { + id: 'srchSnp1', + kind: 'search', + mode: 'snapshot', + user_id: 'user0001', + created_at: '2026-09-02T10:00:00Z', + title: 'debt in Ghana', + data: { query: 'debt in Ghana' }, + parts: [{ ...SEARCH_PART, part_index: 0, content_version: 'cv_7', artifact_digest: null }], + }, + // A snapshot canvas with no mode: a pinned card from turn 1, a removed + // card, a ref-less card with inline numbers, a card on a derived frame, + // and an agent note. + { + id: 'canvSnp1', + kind: 'canvas', + mode: null, + user_id: 'user0001', + created_at: '2026-09-03T10:00:00Z', + title: 'Aid to Kenya', + data: canvas( + [ + chart('c0', 1, { ref: ref(1), frame: FRAME }), + chart('gone', 2, { ref: ref(2), frame: FRAME }), + table('c1', 2), + chart('c2', 2, { ref: ref(3), frame: DERIVED, horizontal: true }), + note('n1', 2), + ], + { pinnedIds: ['c0'], removedIds: ['gone'] }, + ), + parts: [], + }, + // A live canvas: each data card it shows has its ref and frame, and an + // older turn's unpinned card is not shown. + { + id: 'canvLiv1', + kind: 'canvas', + mode: 'live', + user_id: 'user0001', + created_at: '2026-09-04T10:00:00Z', + title: 'Live aid', + data: canvas([ + table('old', 1, { ref: ref(7), frame: FRAME }), + table('c1', 2, { ref: ref(4), frame: FRAME }), + table('c2', 2, { ref: ref(4), frame: FRAME }), + ]), + parts: [], + }, + // Two shares made at the same moment leave by id. + { + id: 'cardSnp2', + kind: 'card', + mode: 'snapshot', + user_id: null, + created_at: '2026-09-05T10:00:00Z', + title: 'Debt', + data: canvas([table('c1', 1)], { pinnedIds: ['c1'], textCards: [] }), + parts: [], + }, + { + id: 'cardLiv1', + kind: 'card', + mode: 'live', + user_id: 'user0001', + created_at: '2026-09-05T10:00:00Z', + title: 'Live card', + data: canvas([chart('c1', 1, { ref: ref(5), frame: FRAME })], { + pinnedIds: ['c1'], + textCards: [], + }), + parts: [], + }, + // A conversation share with no mode: a snapshot, carried whole. + { + id: 'convSnp1', + kind: 'conversation', + mode: null, + user_id: 'user0001', + created_at: '2026-09-06T10:00:00Z', + title: 'Aid to Kenya', + data: canvas([chart('c1', 1, { ref: ref(6), frame: FRAME })], { + messages: [{ role: 'user', text: 'aid to Kenya' }], + }), + parts: [], + }, +] satisfies StoredShare[]; + +// The stored copy of a Studio row of ROWS. +const canvasOf = (index: number) => ROWS[index]!.data as ReturnType; + +const held = (data: ReturnType, widgets: unknown[]) => ({ + ...data, + sessionId: null, + componentRefs: [], + widgets, +}); + +const EXPECTED = [ + { + id: 'srchLiv1', + kind: 'search', + mode: 'live', + owner: null, + created_at: '2026-09-01T10:00:00.000001Z', + title: 'aid to Kenya', + slots: [], + payload: { + query: 'aid to Kenya', + parts: [ROWS[0].parts[1], ROWS[0].parts[0]], + }, + }, + { + id: 'srchSnp1', + kind: 'search', + mode: 'snapshot', + owner: 'user0001', + created_at: '2026-09-02T10:00:00.000000Z', + title: 'debt in Ghana', + slots: [], + payload: { query: 'debt in Ghana', parts: ROWS[1].parts }, + }, + { + id: 'canvSnp1', + kind: 'canvas', + mode: 'snapshot', + owner: 'user0001', + created_at: '2026-09-03T10:00:00.000000Z', + title: 'Aid to Kenya', + slots: [ + { ...ref(1), frame: FRAME }, + { ...ref(3), frame: DERIVED }, + ], + payload: held(canvasOf(2), [ + { kind: 'chart', id: 'c0', turn: 1, chartType: 'line', slot: 0 }, + canvasOf(2).widgets[1], + canvasOf(2).widgets[2], + { kind: 'chart', id: 'c2', turn: 2, chartType: 'line', horizontal: true, slot: 1 }, + canvasOf(2).widgets[4], + ]), + }, + { + id: 'canvLiv1', + kind: 'canvas', + mode: 'live', + owner: 'user0001', + created_at: '2026-09-04T10:00:00.000000Z', + title: 'Live aid', + slots: [ + { ...ref(4), frame: FRAME }, + { ...ref(4), frame: FRAME }, + ], + payload: held(canvasOf(3), [ + { kind: 'table', id: 'old', turn: 1 }, + { kind: 'table', id: 'c1', turn: 2, slot: 0 }, + { kind: 'table', id: 'c2', turn: 2, slot: 1 }, + ]), + }, + { + id: 'cardLiv1', + kind: 'card', + mode: 'live', + owner: 'user0001', + created_at: '2026-09-05T10:00:00.000000Z', + title: 'Live card', + slots: [{ ...ref(5), frame: FRAME }], + payload: held(canvasOf(5), [ + { kind: 'chart', id: 'c1', turn: 1, chartType: 'line', slot: 0 }, + ]), + }, + { + id: 'cardSnp2', + kind: 'card', + mode: 'snapshot', + owner: null, + created_at: '2026-09-05T10:00:00.000000Z', + title: 'Debt', + slots: [], + payload: held(canvasOf(4), canvasOf(4).widgets), + }, + { + id: 'convSnp1', + kind: 'conversation', + mode: 'snapshot', + owner: 'user0001', + created_at: '2026-09-06T10:00:00.000000Z', + title: 'Aid to Kenya', + slots: [], + payload: { ...canvasOf(6), sessionId: null, componentRefs: [] }, + }, +]; + +/** + * The export over a migrated database holding one share per legacy case, + * read as the runtime role: each lands with its mode, owner, slots and + * payload, in `created_at` then id order, from `--since` on; and the + * script writes the same lines, or refuses with nothing written. + */ +test('every legacy share case lands in the export', async (t) => { + const database = await createTestDatabase(); + t.after(() => database.teardown()); + + await applyMigrations(database.migratorPool(), { + migrationsFolder: MIGRATIONS_FOLDER, + runtimeDbUser: database.runtimeUser, + migratorDbUser: database.migratorUser, + }); + + const pool = database.runtimePool(); + + await pool.query(`insert into "user" (id, name, email) values ('user0001', 'A', 'a@one.org')`); + + for (const row of ROWS) { + await pool.query( + `insert into share (id, user_id, kind, mode, title, data, created_at) + values ($1, $2, $3, $4, $5, $6::jsonb, $7)`, + [row.id, row.user_id, row.kind, row.mode, row.title, row.data, row.created_at], + ); + + for (const part of row.parts) { + await pool.query( + `insert into share_ref (share_id, part_index, spec_id, identity, entity_groups, + variable_text, content_version, artifact_digest, operations, assumptions) + values ($1, $2, $3, $4::jsonb, $5::jsonb, $6, $7, $8, $9::jsonb, $10::jsonb)`, + [ + row.id, + part.part_index, + part.spec_id, + JSON.stringify(part.identity), + JSON.stringify(part.entity_groups), + part.variable_text, + part.content_version, + part.artifact_digest, + JSON.stringify(part.operations), + JSON.stringify(part.assumptions), + ], + ); + } + } + + // A live refresh cache row, which the export never reads. + await pool.query( + `insert into share_card_result (share_id, card_id, status) values ('canvLiv1', 'c1', 'current')`, + ); + + const all = exportShares(await readShares(pool, '2026-08-01T00:00:00Z')); + + // The lines are JSON, where a member left undefined is absent. + assert.deepEqual(JSON.parse(JSON.stringify(all)), { lines: EXPECTED, refusals: [] }); + + const later = exportShares(await readShares(pool, '2026-09-05T10:00:00+00:00')); + + assert.deepEqual( + later.lines.map(({ id }) => id), + ['cardLiv1', 'cardSnp2', 'convSnp1'], + ); + + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'share-export-')); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + + const out = path.join(dir, 'shares.jsonl'); + const env = { + ...process.env, + DATABASE_URL: `postgres://${database.runtimeUser}:${database.runtimePassword}@${process.env.PGHOST ?? 'localhost'}:${process.env.PGPORT ?? 5433}/${database.databaseName}`, + }; + const exported = await run( + process.execPath, + [SCRIPT, '--since', '2026-08-01T00:00:00Z', '--out', out], + { env }, + ); + + const written = await fs.readFile(out, 'utf8'); + + assert.ok(written.endsWith('\n')); + assert.deepEqual( + written + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line)), + EXPECTED, + ); + assert.match(exported.stderr, /^7 shares exported since 2026-08-01T00:00:00Z: /m); + + await pool.query( + `insert into share (id, kind, title, data, created_at) + values ('gallery1', 'gallery', 'G', '{}'::jsonb, '2026-09-07T10:00:00Z')`, + ); + + const refusedOut = path.join(dir, 'refused.jsonl'); + const refused = await run( + process.execPath, + [SCRIPT, '--since', '2026-08-01T00:00:00Z', '--out', refusedOut], + { env }, + ).catch((error: { code: number; stderr: string }) => error); + + assert.equal('code' in refused && refused.code, 1); + assert.match(refused.stderr, /share gallery1: kind "gallery" is not a share kind/); + await assert.rejects(fs.access(refusedOut)); + + const noOffset = await run(process.execPath, [SCRIPT, '--since', '2026-08-01T00:00:00'], { + env, + }).catch((error: { code: number }) => error); + + assert.equal('code' in noOffset && noOffset.code, 2); + + // A share's parts deleted between the read of the shares and the read of + // their parts still export with it, as both reads see one snapshot. + let raced = false; + const read = await readShares( + afterFirstSelect(pool, async () => { + raced = true; + await database.adminPool().query(`delete from share_ref where share_id = 'srchSnp1'`); + }), + '2026-09-02T00:00:00Z', + ); + + assert.ok(raced); + assert.equal(read.find(({ id }) => id === 'srchSnp1')?.parts.length, 1); +}); + +// `pool`, with `between` run once, after the first SELECT through the pool +// or one of its clients has answered. +const afterFirstSelect = (pool: Pool, between: () => Promise): Pool => { + let fired = false; + const watched = (target: T): T => + new Proxy(target, { + get(object, key) { + const value: unknown = Reflect.get(object, key); + + if (key === 'connect') { + return async () => watched(await (object as Pool).connect()); + } + + if (key !== 'query') { + return typeof value === 'function' ? value.bind(object) : value; + } + + return async (text: string, values?: unknown[]) => { + const result = await object.query(text, values); + + if (!fired && /^\s*select/i.test(text)) { + fired = true; + await between(); + } + + return result; + }; + }, + }); + + return watched(pool); +}; + +const stored = (overrides: Partial): StoredShare => ({ + id: 'abcd1234', + kind: 'canvas', + mode: 'live', + user_id: null, + created_at: '2026-09-01T00:00:00.000000Z', + title: 'T', + data: canvas([]), + parts: [], + ...overrides, +}); + +test('a share the export cannot place is refused with its reason', () => { + const cases: [Partial, string][] = [ + [{ id: 'short' }, 'the id is not 8 letters and digits'], + [{ mode: 'pinned' }, 'mode "pinned" is not live or snapshot'], + [{ kind: 'conversation', mode: 'live' }, 'a conversation share cannot be live'], + [{ data: { widgets: [] } }, 'data has no widgets, pinnedIds and removedIds arrays'], + [{ data: canvas([table('c1', 1)]) }, "live card c1 has no component ref in dfo-apis' form"], + [ + { + data: canvas([ + table('c1', 1, { ref: { component_id: '../x', revision: 1 }, frame: FRAME }), + ]), + }, + "live card c1 has no component ref in dfo-apis' form", + ], + [ + { + parts: [ + { ...SEARCH_PART, part_index: 0, content_version: null, artifact_digest: null }, + ], + }, + 'a canvas share has 1 share_ref rows', + ], + [ + { + kind: 'search', + mode: 'snapshot', + data: { query: 'q' }, + parts: [ + { ...SEARCH_PART, part_index: 0, content_version: null, artifact_digest: null }, + ], + }, + 'snapshot part 0 has no content_version', + ], + ]; + + for (const [overrides, refusal] of cases) { + assert.deepEqual(placeShare(stored(overrides)), { refusal }, JSON.stringify(overrides)); + } +}); + +test('a card with a ref and no frame the website holds gets a slot with a null frame', () => { + for (const mode of ['live', 'snapshot']) { + const placed = placeShare( + stored({ + mode, + data: canvas([ + table('c1', 1, { ref: ref(1) }), + table('c2', 1, { ref: ref(2), frame: { spec_id: 'spec_aid' } }), + ]), + }), + ); + + assert.ok('line' in placed, mode); + assert.deepEqual( + placed.line.slots, + [ + { ...ref(1), frame: null }, + { ...ref(2), frame: null }, + ], + mode, + ); + assert.deepEqual( + placed.line.payload.widgets, + [ + { kind: 'table', id: 'c1', turn: 1, slot: 0 }, + { kind: 'table', id: 'c2', turn: 1, slot: 1 }, + ], + mode, + ); + } +}); diff --git a/db/test/share-mode.test.ts b/db/test/share-mode.test.ts index dde778f..81f16fc 100644 --- a/db/test/share-mode.test.ts +++ b/db/test/share-mode.test.ts @@ -4,13 +4,12 @@ import test from 'node:test'; import { eq } from 'drizzle-orm'; import { drizzle } from 'drizzle-orm/node-postgres'; import type { SavedConversation } from '../../src/lib/chat/history.ts'; -import { GUEST_SHARES, isLiveStudioShare, searchShareMode } from '../../src/lib/chat/shareMode.ts'; +import { LIVE_CARD_TITLE } from '../../src/lib/chat/sharePayload.ts'; +import { isLiveStudioShare, searchShareMode } from '../../src/lib/chat/shareMode.ts'; import { deleteOwnedShare, insertStudioShare, - LIVE_CARD_TITLE, snapshotOwnedShare, - spendGuestShare, } from '../../src/lib/chat/shareWrites.ts'; import { applyMigrations } from '../apply-migrations.ts'; import { share, user } from '../schema.ts'; @@ -200,17 +199,6 @@ test('Studio share writes', async (t) => { assert.equal((await stored(id))?.mode, 'live'); }); - await t.test('a guest network gets GUEST_SHARES a day, counted by /64', async () => { - const spends = await Promise.all( - Array.from({ length: GUEST_SHARES + 1 }, (_, i) => - spendGuestShare(db, `2001:db8:1:1::${i + 1}`), - ), - ); - - assert.equal(spends.filter(Boolean).length, GUEST_SHARES); - assert.equal(await spendGuestShare(db, '2001:db8:1:2::1'), true); - }); - await t.test('a search share has no owner writes', async () => { await db.insert(share).values([ { diff --git a/db/test/share-token.test.ts b/db/test/share-token.test.ts new file mode 100644 index 0000000..37f0604 --- /dev/null +++ b/db/test/share-token.test.ts @@ -0,0 +1,249 @@ +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { drizzle } from 'drizzle-orm/node-postgres'; + +import { + deleteShareToken, + fenceShareTokens, + putShareToken, + readShareToken, + refreshFence, + shareTokensFenced, + unfenceShareTokens, +} from '../../src/lib/chat/shareTokens.ts'; +import { applyMigrations } from '../apply-migrations.ts'; +import { importShareTokens, readTokenLines } from '../share-token-import.ts'; +import { user } from '../schema.ts'; +import { createTestDatabase } from './harness.ts'; + +const MIGRATIONS_FOLDER = path.resolve(import.meta.dirname, '../migrations'); +const SCRIPT = path.resolve(import.meta.dirname, '../../scripts/import-share-tokens.ts'); +const run = promisify(execFile); + +/** + * `shareTokens.ts` against the migrated tables as the runtime role: a user + * reads back the token they put and replaces it, another user neither reads + * nor overwrites it, and deleting the user deletes their tokens. + */ +test('share write tokens', async (t) => { + const database = await createTestDatabase(); + t.after(() => database.teardown()); + + await applyMigrations(database.migratorPool(), { + migrationsFolder: MIGRATIONS_FOLDER, + runtimeDbUser: database.runtimeUser, + migratorDbUser: database.migratorUser, + }); + + const db = drizzle(database.runtimePool()); + + await db.insert(user).values([ + { id: 'owner', name: 'O', email: 'o@one.org' }, + { id: 'other', name: 'X', email: 'x@one.org' }, + ]); + + const owner = { shareId: 'Ab3dEf7h', userId: 'owner' }; + const other = { shareId: 'Ab3dEf7h', userId: 'other' }; + + assert.equal(await readShareToken(db, owner), null); + assert.equal(await putShareToken(db, { ...owner, writeToken: 'shw_one' }), true); + assert.equal(await readShareToken(db, owner), 'shw_one'); + assert.equal(await putShareToken(db, { ...owner, writeToken: 'shw_two' }), true); + assert.equal(await readShareToken(db, owner), 'shw_two'); + + // Putting the token already held changes nothing; a new token restamps the row. + const createdAt = async () => + ( + await database + .runtimePool() + .query<{ created_at: string }>( + 'select created_at::text from share_token where share_id = $1', + [owner.shareId], + ) + ).rows.map((row) => row.created_at); + const stamped = await createdAt(); + + assert.equal(await putShareToken(db, { ...owner, writeToken: 'shw_two' }), true); + assert.deepEqual(await createdAt(), stamped); + assert.equal(await putShareToken(db, { ...owner, writeToken: 'shw_three' }), true); + assert.notDeepEqual(await createdAt(), stamped); + assert.equal(await putShareToken(db, { ...owner, writeToken: 'shw_two' }), true); + + assert.equal(await readShareToken(db, other), null); + assert.equal(await putShareToken(db, { ...other, writeToken: 'shw_theirs' }), false); + assert.equal(await readShareToken(db, owner), 'shw_two'); + + await deleteShareToken(db, other); + assert.equal(await readShareToken(db, owner), 'shw_two'); + await deleteShareToken(db, owner); + assert.equal(await readShareToken(db, owner), null); + assert.equal(await putShareToken(db, { ...owner, writeToken: 'shw_two' }), true); + + // A fenced account, or one gone, stores no token. + await fenceShareTokens(db, 'other'); + assert.equal( + await putShareToken(db, { ...other, shareId: 'Zz9yXw8v', writeToken: 'shw_x' }), + false, + ); + // A fence older than 15 minutes is one a failed deletion left behind. + await database + .runtimePool() + .query(`update "user" set deleting_at = now() - interval '16 minutes' where id = 'other'`); + assert.equal( + await putShareToken(db, { ...other, shareId: 'Zz9yXw8v', writeToken: 'shw_x' }), + true, + ); + await deleteShareToken(db, { ...other, shareId: 'Zz9yXw8v' }); + + // An attempt lifts only its own fence, and sets afresh only its own. + const first = await fenceShareTokens(db, 'other'); + const second = await fenceShareTokens(db, 'other'); + + assert.notEqual(first, second); + assert.equal(await refreshFence(db, { userId: 'other', fence: first }), null); + await unfenceShareTokens(db, { userId: 'other', fence: first }); + assert.equal(await shareTokensFenced(db, 'other'), true); + + const refreshed = await refreshFence(db, { userId: 'other', fence: second }); + + assert.ok(refreshed !== null && refreshed !== second); + await unfenceShareTokens(db, { userId: 'other', fence: refreshed }); + assert.equal(await shareTokensFenced(db, 'other'), false); + + await database.runtimePool().query(`delete from "user" where id = 'owner'`); + assert.equal( + await putShareToken(db, { ...owner, shareId: 'Zz9yXw8v', writeToken: 'shw_x' }), + false, + ); + assert.equal(await putShareToken(db, { ...other, writeToken: 'shw_theirs' }), true); + assert.equal(await readShareToken(db, other), 'shw_theirs'); +}); + +/** + * scripts/import-share-tokens.ts over a migrated database: a run stores each + * signed-in owner's token, a second run of the same file stores the same + * rows again, a share whose owner is gone is skipped, and a file with an + * unreadable line stores nothing. + */ +test("the token import stores each owner's token once, however often it runs", async (t) => { + const database = await createTestDatabase(); + t.after(() => database.teardown()); + + await applyMigrations(database.migratorPool(), { + migrationsFolder: MIGRATIONS_FOLDER, + runtimeDbUser: database.runtimeUser, + migratorDbUser: database.migratorUser, + }); + + const pool = database.runtimePool(); + + await pool.query( + `insert into "user" (id, name, email) values ('owner', 'O', 'o@one.org'), ('other', 'X', 'x@one.org')`, + ); + + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'share-tokens-')); + t.after(() => fs.rm(dir, { recursive: true, force: true })); + + const file = path.join(dir, 'tokens.jsonl'); + const lines = [ + { id: 'Ab3dEf7h', owner: 'owner', write_token: 'shw_a' }, + { id: 'Zy9xWv8u', owner: 'other', write_token: 'shw_b' }, + { id: 'Gone0001', owner: 'deleted', write_token: 'shw_c' }, + ]; + + await fs.writeFile(file, `${lines.map((line) => JSON.stringify(line)).join('\n')}\n`); + + const env = { + ...process.env, + DATABASE_URL: `postgres://${database.runtimeUser}:${database.runtimePassword}@${process.env.PGHOST ?? 'localhost'}:${process.env.PGPORT ?? 5433}/${database.databaseName}`, + }; + const tokens = async () => + ( + await pool.query<{ share_id: string; user_id: string; write_token: string }>( + 'select share_id, user_id, write_token from share_token order by share_id', + ) + ).rows.map(({ share_id: id, user_id: owner, write_token: token }) => [id, owner, token]); + const stored = [ + ['Ab3dEf7h', 'owner', 'shw_a'], + ['Zy9xWv8u', 'other', 'shw_b'], + ]; + + await run(process.execPath, [SCRIPT, '--in', file], { env }); + assert.deepEqual(await tokens(), stored); + + // A second import of the same lines stores the same rows and skips the + // share whose owner is gone. + assert.deepEqual(await importShareTokens(drizzle(pool), lines), { + stored: 2, + ownerGone: ['Gone0001'], + ownerDeleting: [], + heldByOther: [], + }); + + // An owner whose account is being deleted stores no token, and is named + // apart from a token another user holds. + const fence = await fenceShareTokens(drizzle(pool), 'other'); + + assert.deepEqual( + await importShareTokens(drizzle(pool), [ + { id: 'New00002', owner: 'other', write_token: 'shw_e' }, + ]), + { stored: 0, ownerGone: [], ownerDeleting: ['New00002'], heldByOther: [] }, + ); + await unfenceShareTokens(drizzle(pool), { userId: 'other', fence }); + assert.deepEqual(await tokens(), stored); + + // A token another user holds is left as it was and fails the run. + const clash = path.join(dir, 'clash.jsonl'); + + await fs.writeFile(clash, `${JSON.stringify({ ...lines[0], owner: 'other' })}\n`); + + const refusedClash = await run(process.execPath, [SCRIPT, '--in', clash], { env }).catch( + (error: { code: number; stderr: string }) => error, + ); + + assert.equal('code' in refusedClash && refusedClash.code, 1); + assert.equal( + await readShareToken(drizzle(pool), { shareId: 'Ab3dEf7h', userId: 'owner' }), + 'shw_a', + ); + + // One unreadable line refuses the whole file. + const bad = path.join(dir, 'bad.jsonl'); + + await fs.writeFile( + bad, + `${JSON.stringify({ id: 'New00001', owner: 'owner', write_token: 'shw_d' })}\n{"id": "short"}\n`, + ); + + const refused = await run(process.execPath, [SCRIPT, '--in', bad], { env }).catch( + (error: { code: number; stderr: string }) => error, + ); + + assert.equal('code' in refused && refused.code, 1); + assert.equal( + await readShareToken(drizzle(pool), { shareId: 'New00001', userId: 'owner' }), + null, + ); +}); + +test('a token file line needs an 8-character id, an owner and a shw_ token', () => { + const text = [ + JSON.stringify({ id: 'Ab3dEf7h', owner: 'u', write_token: 'shw_a' }), + '', + JSON.stringify({ id: 'Ab3dEf7', owner: 'u', write_token: 'shw_a' }), + JSON.stringify({ id: 'Ab3dEf7h', owner: '', write_token: 'shw_a' }), + JSON.stringify({ id: 'Ab3dEf7h', owner: 'u', write_token: 'cmw_a' }), + 'not json', + ].join('\n'); + + assert.deepEqual(readTokenLines(text), { + lines: [{ id: 'Ab3dEf7h', owner: 'u', write_token: 'shw_a' }], + invalid: [3, 4, 5, 6], + }); +}); diff --git a/deploy/README.md b/deploy/README.md index 07040e0..c38574b 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -136,6 +136,28 @@ A domain in `webmail_domain` is `Personal email` whatever its count. The table h A read with no rows fails the run and leaves the table as it was while `webmail_domain` is empty, since the view then holds no rows and webmail domains would show by name, and while the user table holds no account, which is a wrong database or an emptied table. When accounts exist and every one is from today, the run replaces the table with no rows, so a deleted account and the organisation it named leave the export. The view `account_daily_source` tells the cases apart, and the Job reads it in the same transaction as `account_daily`. +## Shares left for cleanup + +Each signed-in owner's share in dfo-apis has its write token in `share_token`. When the website cannot take such a share down, it writes the share id, the owner, the token and dfo-apis' refusal code to `share_cleanup` and logs `share_cleanup_recorded` with the share id at error level. Two paths write a row: an account deletion whose take-down dfo-apis refuses with a `write_token_*` code (`takeDownUserShares` in `src/lib/chat/apisShares.ts`), and a new share whose token could not be stored and whose take-down failed too (`createOwnedShare`). The share stays public until an operator takes it down. The rows outlive the user. `share_cleanup_recorded` is the alert hook, so alert on that text and keep it unchanged in the code. A row the website could not write is logged as `share_cleanup_not_recorded`, with the share id, and needs the same take-down. + +To drain the table, list its rows against the website database: + +```sql +select share_id, user_id, reason, recorded_at from share_cleanup order by recorded_at; +``` + +For each row, take the share down in dfo-apis with its maintenance command: + +```bash +datasets-maintenance take-down-share +``` + +Then delete the row: + +```sql +delete from share_cleanup where share_id = ''; +``` + ## GitHub vars and secrets `build-migrate-deploy.yml` reads these from a GitHub Environment matching the @@ -197,7 +219,8 @@ The app defines the definitive contract once it exists. Known so far: | `STUDIO_TOKEN` | Bearer token the studio proxy sends to the harness; secret reference set by infra | | `SEARCH_API_BASE` | The `dfo-apis` Search, Datasets and graph APIs; required, a production server without it refuses to start; plain env var set by infra | | `MCP_API_TOKEN` | Bearer token the `/mcp` proxy sends to the harness; infra mounts it from Secret Manager (`dfo-dev-mcp-api-token`, the harness's own secret) | -| `CALLER_HMAC_KEY` | HMAC key for the client-IP digest `/mcp` sends as `x-dfo-caller`; unset, none is sent; secret set by infra | +| `CALLER_HMAC_KEY` | HMAC key for the client digest sent as `x-dfo-caller` (`/mcp`, and dfo-apis beside the principal); unset, none is sent; secret set by infra | +| `DFO_CALLER_PRINCIPAL` | Secret shared with dfo-apis (its `DFO_CALLER_PRINCIPAL`), sent as `x-dfo-caller-principal` on every dfo-apis call so dfo-apis trusts `x-dfo-caller`; unset, neither is sent | | `RATE_LIMIT_WINDOW_S` | Rate-limit window in seconds; default 60 | | `RATE_LIMIT_API` | Calls per window per address on `/api`; default 120 | | `RATE_LIMIT_PIN` | Calls per window per address on `/api/download-pin`; default 10 | @@ -220,7 +243,12 @@ The proxy forwards `content-type`, `accept` and every `mcp-*` request header (`mcp-method` and `mcp-name` among them, which the harness checks against the request body), and returns `content-type`, `content-disposition`, `content-length`, `www-authenticate` and every `mcp-*` response header. -`x-dfo-caller` is set by the proxy, whatever the client sent. `authorization` +`x-dfo-caller` is set by the proxy, whatever the client sent. A call to +dfo-apis carries `x-dfo-caller` beside `x-dfo-caller-principal` when +`DFO_CALLER_PRINCIPAL` is set, and neither when it is unset. The digest's +input is the client address folded as the rate limiter folds it (an IPv6 +address as its /64), so dfo-apis, which computes the same digest for a direct +caller, keys both on one allowance. `authorization` is set by the proxy only when `MCP_API_TOKEN` is set; otherwise the client's own `authorization` header goes through to the harness. A malformed `TRUSTED_PROXY_HOPS` stops a production server at startup. If one reaches a diff --git a/docs/analytics.md b/docs/analytics.md index c3df018..404f805 100644 --- a/docs/analytics.md +++ b/docs/analytics.md @@ -100,7 +100,7 @@ The site writes one JSON line to stdout for each logged request, which Cloud Log {"severity":"INFO","message":"request","caller":"ip:3f9c0a…","method":"GET","path":"/embed/abc123","host":"developmentfinance.org","kind":"page","status":200,"finished":true,"bot":false,"prerender":false} ``` -`caller` is the same `ip:` the `/mcp` proxy sends upstream as `x-dfo-caller` (`src/lib/proxy/caller.ts`): an HMAC of the client address under `CALLER_HMAC_KEY`, with the address taken from `X-Forwarded-For` at the `TRUSTED_PROXY_HOPS` position. It is null when the key is unset or the request carries no usable address. `path` is the request target as it arrived, captured when the request starts so a later rewrite does not change it. It has no query string or fragment, and dot segments are resolved, so `/a/../mcp` is logged as `/mcp`. A target such as `//mcp` is read as a path and logged as `//mcp`. An http or https absolute-form target such as `http://host/path` is logged as its pathname. Any other target, such as one with another scheme (`ftp://`, `file:`, `mailto:`), an authority-form `host:port`, or an http or https URL that does not parse, is logged as the fixed path `/:unparsable`, since it can hold a user name, a password or an IP address. The asterisk-form target of `OPTIONS *` is logged as `*`, with kind `page`. `kind` is `api` for `/api`, `/mcp`, `/openapi.json` and `/codes` and every path below each of them (`/apiary` is not one). That covers the site's own endpoints under `/api`, such as auth, events and share, as well as the platform proxy's routes. `kind` is `prefetch` for a Next.js prefetch (`Next-Router-Prefetch`) or a browser prefetch (`Sec-Purpose: prefetch`), `rsc` for a client-side navigation (`RSC: 1`) and `page` for everything else. A `Sec-Purpose` header does not change the kind of an RSC fetch or a Next.js prefetch. Embeds under `/embed/` are logged as pages. `/_next/` files, images and fonts are not logged, and are dropped before the line's headers are read or its caller computed. `status` is the response's HTTP status code. `finished` is true when the response ended normally and false when it closed first, so a status of 200 with `finished` false is a visitor who left a stream. A response that closes before the app sent a status has none of its own, so the line carries 499 (client closed request) with `finished` false. The 499 means the app sent no status before the connection closed. It also covers a response Node answered itself, such as a 408 on a request timeout or a 400 on a client error, which never passes through the app. A response that called `writeHead` but wrote nothing to the socket before the close is logged with that status and `finished` false. Infra's parse keeps the line and counts it as a 4xx, outside the readers' 2xx count. Infra's `website_requests_daily` counts a line whose response did not finish like a finished one. `prerender` is true for a document request (no `RSC` and no `Next-Router-Prefetch` header) with `Sec-Purpose: prefetch;prerender`: the browser renders the page ahead of time and shows it on activation with no second request, so that one line is the page view. It has kind `page`, which keeps infra's `website_requests_daily` counting it as a parsed page request. The line is a page view whether or not the visitor opens the page, because the server cannot see activation. `bot` is true when the `User-Agent` header is missing or empty, or when the `isbot` package flags it: crawlers such as Googlebot, bingbot and ClaudeBot, link previews such as Slack's and Discord's, headless browsers and tools such as curl. A client that sends a browser's user agent counts as a reader. The line holds no raw IP address, query string, cookie, `Authorization` header or user agent. +`caller` is the same `ip:` the `/mcp` proxy sends upstream as `x-dfo-caller` (`src/lib/proxy/caller.ts`): an HMAC of the client address (an IPv6 address as its /64) under `CALLER_HMAC_KEY`, with the address taken from `X-Forwarded-For` at the `TRUSTED_PROXY_HOPS` position. It is null when the key is unset or the request carries no usable address. `path` is the request target as it arrived, captured when the request starts so a later rewrite does not change it. It has no query string or fragment, and dot segments are resolved, so `/a/../mcp` is logged as `/mcp`. A target such as `//mcp` is read as a path and logged as `//mcp`. An http or https absolute-form target such as `http://host/path` is logged as its pathname. Any other target, such as one with another scheme (`ftp://`, `file:`, `mailto:`), an authority-form `host:port`, or an http or https URL that does not parse, is logged as the fixed path `/:unparsable`, since it can hold a user name, a password or an IP address. The asterisk-form target of `OPTIONS *` is logged as `*`, with kind `page`. `kind` is `api` for `/api`, `/mcp`, `/openapi.json` and `/codes` and every path below each of them (`/apiary` is not one). That covers the site's own endpoints under `/api`, such as auth, events and share, as well as the platform proxy's routes. `kind` is `prefetch` for a Next.js prefetch (`Next-Router-Prefetch`) or a browser prefetch (`Sec-Purpose: prefetch`), `rsc` for a client-side navigation (`RSC: 1`) and `page` for everything else. A `Sec-Purpose` header does not change the kind of an RSC fetch or a Next.js prefetch. Embeds under `/embed/` are logged as pages. `/_next/` files, images and fonts are not logged, and are dropped before the line's headers are read or its caller computed. `status` is the response's HTTP status code. `finished` is true when the response ended normally and false when it closed first, so a status of 200 with `finished` false is a visitor who left a stream. A response that closes before the app sent a status has none of its own, so the line carries 499 (client closed request) with `finished` false. The 499 means the app sent no status before the connection closed. It also covers a response Node answered itself, such as a 408 on a request timeout or a 400 on a client error, which never passes through the app. A response that called `writeHead` but wrote nothing to the socket before the close is logged with that status and `finished` false. Infra's parse keeps the line and counts it as a 4xx, outside the readers' 2xx count. Infra's `website_requests_daily` counts a line whose response did not finish like a finished one. `prerender` is true for a document request (no `RSC` and no `Next-Router-Prefetch` header) with `Sec-Purpose: prefetch;prerender`: the browser renders the page ahead of time and shows it on activation with no second request, so that one line is the page view. It has kind `page`, which keeps infra's `website_requests_daily` counting it as a parsed page request. The line is a page view whether or not the visitor opens the page, because the server cannot see activation. `bot` is true when the `User-Agent` header is missing or empty, or when the `isbot` package flags it: crawlers such as Googlebot, bingbot and ClaudeBot, link previews such as Slack's and Discord's, headless browsers and tools such as curl. A client that sends a browser's user agent counts as a reader. The line holds no raw IP address, query string, cookie, `Authorization` header or user agent. The line reads the request's raw headers, which keep the `RSC` and `Next-Router-Prefetch` headers that Next.js removes from `request.headers`. Logging leaves request bodies unread, so API and MCP payloads stream to their handlers in full. A Next.js proxy would buffer those bodies and truncate them at its default 10 MB limit. diff --git a/docs/development.md b/docs/development.md index da437ac..9e44e84 100644 --- a/docs/development.md +++ b/docs/development.md @@ -32,11 +32,9 @@ at a mail catcher. To open the dev server from a phone/tablet on the same network, put the machine's LAN IP in `DEV_ORIGINS`. -Live Studio shares need `LIVE_REFRESH_PRINCIPAL`, the credential dfo-apis -and dfo-ai ask of a card refresh, set to the same value on all three. With -it empty the Share dialog offers Snapshot only. A live card's refreshed -answer stands for `LIVE_SHARE_WINDOW_S` seconds (default 300, at most 900) -before the next viewer asks the back ends again. +A live Studio share draws each card from dfo-apis' public slot card route +(`GET /api/presentation/shares/{id}/slots/{n}`), which checks for newer data +at most once per 15 minutes and takes no credential. ### Reach the dev APIs diff --git a/docs/share-export.md b/docs/share-export.md new file mode 100644 index 0000000..0410270 --- /dev/null +++ b/docs/share-export.md @@ -0,0 +1,82 @@ +# Share export + +`scripts/export-shares.ts` writes the website's shares as JSON lines, one share per line, and dfo-apis' `maintenance.py import-shares` reads them. The import keeps each 8-character id, so every `/s/` and `/embed/` link keeps working. The queries and the rule that places each share are in `db/share-export.ts`. + +``` +node scripts/export-shares.ts --since 2026-10-01T00:00:00Z --out shares.jsonl +``` + +The script reads the database from `DATABASE_URL`, or from the `WEBSITE_DB_*` Cloud SQL variables the runtime uses, and writes nothing to it. `--since` takes an ISO timestamp with a `Z` or `+hh:mm` offset and exports every share whose `created_at` is at or after it. Without `--out` the lines go to stdout. A count by kind and mode goes to stderr. A bad `--since` exits 2. + +A share the script cannot place fails the run. The script prints `share : ` for each such share to stderr, exits 1, and writes no output, so a file on disk is always a whole export. Runs overlap safely, because the import is idempotent and a later run with an earlier `--since` writes the same lines for the same shares. + +## Lines + +Each line is one JSON object with these members, and lines are ordered by `created_at`, then by `id`. + +- `id`: the share id, 8 characters from `[A-Za-z0-9]`, case-sensitive. +- `kind`: `search`, `conversation`, `canvas`, or `card`. +- `mode`: `live` or `snapshot`. A share stored with no mode gets the default the website applies: `live` for a search share (`searchShareMode` in `src/lib/chat/shareMode.ts`) and `snapshot` for the other kinds (`isLiveStudioShare`). +- `owner`: the website user id of the share's maker, or null for a guest's share. A guest's share imports with no write token, and nobody can change or delete it. +- `created_at`: the stored moment in UTC to the microsecond, as `2026-09-01T10:00:00.000001Z`. +- `title`: the stored title. +- `slots`: the data cards dfo-apis holds, in display order, each `{"component_id": "cmp_…", "revision": 3, "frame": {"spec_id": "spec_…" or "ops_…", "content_version": "…"}}`. `frame` is `null` when the stored card names no frame in that form, and the import reads the frame from the component revision. Only canvas and card shares have slots. Search and conversation shares have `[]`. +- `payload`: website-owned JSON that dfo-apis stores as given and does not read. It holds what the website needs to render the share beyond its slots. + +## Where each stored share lands + +| Stored share | `mode` | `slots` | `payload` | +| --------------------------------------- | ---------- | -------------------------------------------------------- | ------------------------------------------------------------------- | +| Search, mode `live` or null | `live` | `[]` | `query` and `parts` | +| Search, mode `snapshot` | `snapshot` | `[]` | `query` and `parts`, each part with its `content_version` | +| Canvas or card, mode `live` | `live` | every data card on the canvas | the stored copy, slotted cards held | +| Canvas or card, mode `snapshot` or null | `snapshot` | every data card on the canvas with a component reference | the stored copy, slotted cards held, the other data cards as stored | +| Conversation, mode `snapshot` or null | `snapshot` | `[]` | the stored copy | + +A search share's `query` is `share.data.query`, or the title when the row has none, as `getSearchShare` reads it. Its `parts` are the share's `share_ref` rows in `part_index` order, each with the members `part_index`, `spec_id`, `identity`, `entity_groups`, `variable_text`, `content_version`, `artifact_digest`, `operations`, and `assumptions`. A live search share's parts have null `content_version` and `artifact_digest` and replay their definitions. A snapshot made before the website stored `artifact_digest` has a null digest and keeps its `content_version`. + +The stored copy of a canvas, card, or conversation share is its `share.data`, a pruned saved conversation (`pruneForShare` in `src/lib/chat/sharePrune.ts`), with `sessionId` set to null and `componentRefs` to `[]`, as every share read does. It keeps the layout, the user's text cards, the titles, notes and views by card id, the theme, pins and removals, and, for a conversation share, the messages. + +## Slots + +The data cards on a canvas are the cards `canvasWidgetsOf` (`src/lib/chat/canvas.ts`) shows, other than text cards, in that order: the stored `widgets` the canvas displays, in stored order. A card share holds one. A data card has a component reference when its `ref` names a component revision as `cmp_` plus up to 64 letters, digits, `_` or `-`, with a whole revision of 1 or more. Its slot's `frame` is the card's `frame` when that names a `spec_` or `ops_` frame with a non-empty content version, and `null` otherwise. `slotCanvas` in `src/lib/chat/sharePayload.ts` places the cards, for this export and for every share the website makes in dfo-apis. + +Each such card becomes one slot. In the payload's `widgets` the card is held: it keeps `kind`, `id`, `turn`, and for a chart `chartType` and `horizontal`, and gains `slot`, the index of its slot in `slots`. Its numbers, title, source and description come from the slot's component revision. Two cards drawing the same revision get a slot each. + +A snapshot's data card without a component reference, which includes every card stored before cards carried a `ref`, stays in `widgets` as stored, with the numbers it showed inline, and has no slot. Widgets the canvas does not show, such as removed cards and older turns' unpinned cards, stay in `widgets` with no slot, so the payload shows the same cards as the stored copy. In a snapshot they stay as stored. In a live share every data card is held, shown or not, because live share pages show none of the numbers stored with the share. + +## Refused shares + +The script refuses a share whose: + +- id is not 8 letters and digits; +- kind is not one of the four; +- mode is set to a value other than `live` or `snapshot`; +- kind is `conversation` and mode is `live`; +- kind is `conversation`, `canvas`, or `card` and it has `share_ref` rows; +- data, for a canvas or card, lacks `widgets`, `pinnedIds`, or `removedIds` arrays; +- canvas or card is live and a data card on it has no component reference; +- search snapshot has a part with no `content_version`. + +## Left out + +`share_card_result` rows cache live refreshes, and dfo-apis refreshes live slots itself, so the export leaves them out. It leaves out the `conversation_id` and `card_id` columns too, because no share page reads them. + +## Round trip on a local stack + +`scripts/share-migration-round-trip.ts` runs the whole migration on a local website and the dfo-apis local stack. It makes two cards in dfo-apis on the stack's seeded readings, seeds one share per legacy case (a search share with no mode from a guest, a search snapshot, a canvas with no mode holding a card with no component reference, a live canvas holding a card with no frame, a live card, a guest's snapshot card with no component reference, and a conversation with no mode), and runs `export-shares`, dfo-apis' `import-shares` and `import-share-tokens`. It then renders `/s/` and `/embed/` of each share through the dev server and checks what each shows: a search share redirects to `/data?share=` and reads back its mode, query and parts from `/api/share/`, a slotted card draws its component revision, a card with no component reference and a conversation's cards show as stored, and each live slot draws its card from `/api/v1/presentation/shares//slots/`. The owner signs in with a session row and switches their live canvas to snapshot with its imported token, and cannot delete a guest's share. Last, it runs both imports again and checks that dfo-apis holds the same shares and `share_token` the same tokens. + +It needs the stack up and seeded (`just local-stack up` and `just local-stack seed` in dfo-apis), a migrated website database, and the dev server on that database in front of the stack, with the caller principal and HMAC key the stack uses: + +``` +docker run -d --rm --name website-db -e POSTGRES_PASSWORD=postgres -p 127.0.0.1:55433:5432 postgres:18 +PGHOST=127.0.0.1 PGPORT=55433 node db/migrate-local.ts +export DATABASE_URL=postgresql://website_runtime:website_runtime_dev_password@127.0.0.1:55433/website +export BETTER_AUTH_SECRET=$(openssl rand -hex 32) +SEARCH_API_BASE=http://127.0.0.1:58083 BETTER_AUTH_URL=http://127.0.0.1:3000 \ + DFO_CALLER_PRINCIPAL=local-caller-principal CALLER_HMAC_KEY=local-caller-hmac-key \ + RATE_LIMIT_API=100000 pnpm dev -H 127.0.0.1 -p 3000 & +DFO_APIS_DIR=/path/to/dfo-apis node scripts/share-migration-round-trip.ts +``` + +The dev server runs on port 3000 because the stack's share links name `http://127.0.0.1:3000`. `WEBSITE_URL` points the script at another address. Each check prints one line, and the script exits 1 when any fails. Its shares, cards and rows stay in place under fresh ids, so it can run again on the same stack. diff --git a/scripts/export-shares.ts b/scripts/export-shares.ts new file mode 100644 index 0000000..f6b5953 --- /dev/null +++ b/scripts/export-shares.ts @@ -0,0 +1,86 @@ +// Writes every share made at or after --since as the JSON lines +// docs/share-export.md defines, for dfo-apis' `import-shares`. +// db/share-export.ts holds the queries and the placement of each share. +// +// node scripts/export-shares.ts --since 2026-10-01T00:00:00Z # to stdout +// node scripts/export-shares.ts --since 2026-10-01T00:00:00Z --out shares.jsonl +// +// It reads the database from DATABASE_URL, or from the WEBSITE_DB_* Cloud +// SQL variables the runtime uses, and writes nothing to it. A share it +// cannot place fails the run with every such share named on stderr, and +// no output is written. +import fs from 'node:fs/promises'; +import { parseArgs } from 'node:util'; + +import { exportShares, readShares } from '../db/share-export.ts'; +import { openPool } from './open-pool.ts'; + +const USAGE = `usage: node scripts/export-shares.ts --since [--out ] + + --since export shares created at or after this moment, e.g. 2026-10-01T00:00:00Z + --out write the lines to this file instead of stdout`; + +// An offset is required, so the moment does not depend on the session's +// time zone. +const WITH_OFFSET = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:\d{2})$/; + +const main = async (): Promise => { + const { values } = parseArgs({ + options: { + since: { type: 'string' }, + out: { type: 'string' }, + help: { type: 'boolean', default: false }, + }, + }); + const { since } = values; + + if (values.help || !since || !WITH_OFFSET.test(since) || Number.isNaN(Date.parse(since))) { + console.error(USAGE); + process.exitCode = values.help ? 0 : 2; + + return; + } + + const { pool, close } = await openPool(); + + try { + const { lines, refusals } = exportShares(await readShares(pool, since)); + + if (refusals.length) { + for (const { id, reason } of refusals) { + console.error(`share ${id}: ${reason}`); + } + + console.error( + `${refusals.length} shares cannot be placed; nothing written. ` + + 'docs/share-export.md says where each case lands.', + ); + process.exitCode = 1; + + return; + } + + const text = lines.map((line) => `${JSON.stringify(line)}\n`).join(''); + + if (values.out) { + await fs.writeFile(values.out, text); + } else { + process.stdout.write(text); + } + + const counts = Object.entries( + Object.groupBy(lines, ({ kind, mode }) => `${kind} ${mode}`), + ).map(([key, group]) => `${key} ${group?.length ?? 0}`); + + console.error( + `${lines.length} shares exported since ${since}${counts.length ? `: ${counts.join(', ')}` : ''}`, + ); + } finally { + await close(); + } +}; + +main().catch((error: unknown) => { + console.error('ERROR:', error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/scripts/import-share-tokens.ts b/scripts/import-share-tokens.ts new file mode 100644 index 0000000..ff80099 --- /dev/null +++ b/scripts/import-share-tokens.ts @@ -0,0 +1,88 @@ +// Stores the write tokens dfo-apis' `import-shares` minted for signed-in +// owners' shares in share_token, so each owner can still switch and delete +// their shares once dfo-apis holds them. db/share-token-import.ts holds the +// reading and the writes. +// +// node scripts/import-share-tokens.ts --in share-tokens.jsonl +// +// The file is the 0600 JSON lines `import-shares` writes, one +// {"id", "owner", "write_token"} per line. It reads the database from +// DATABASE_URL, or from the WEBSITE_DB_* Cloud SQL variables the runtime +// uses. A run of the same file again changes nothing. A file with a line it +// cannot read is refused whole, with each such line named on stderr, and +// nothing is stored. A share whose owner is no longer a user is skipped. One +// whose owner's account is being deleted stores no token, and one whose token +// another user holds is left as it was; either fails the run. +import fs from 'node:fs/promises'; +import { parseArgs } from 'node:util'; + +import { drizzle } from 'drizzle-orm/node-postgres'; + +import { importShareTokens, readTokenLines } from '../db/share-token-import.ts'; +import { openPool } from './open-pool.ts'; + +const USAGE = `usage: node scripts/import-share-tokens.ts --in + + --in the token file dfo-apis' import-shares wrote`; + +const main = async (): Promise => { + const { values } = parseArgs({ + options: { in: { type: 'string' }, help: { type: 'boolean', default: false } }, + }); + + if (values.help || !values.in) { + console.error(USAGE); + process.exitCode = values.help ? 0 : 2; + + return; + } + + const { lines, invalid } = readTokenLines(await fs.readFile(values.in, 'utf8')); + + if (invalid.length) { + for (const line of invalid) { + console.error(`line ${line}: not {"id", "owner", "write_token"} with a shw_ token`); + } + + console.error(`${invalid.length} lines cannot be read; nothing stored.`); + process.exitCode = 1; + + return; + } + + const { pool, close } = await openPool(); + + try { + const { stored, ownerGone, ownerDeleting, heldByOther } = await importShareTokens( + drizzle(pool), + lines, + ); + + for (const id of ownerGone) { + console.error(`share ${id}: its owner is no longer a user; skipped`); + } + + for (const id of ownerDeleting) { + console.error(`share ${id}: its owner's account is being deleted; not stored`); + } + + for (const id of heldByOther) { + console.error(`share ${id}: another user holds its token; left as it was`); + } + + console.error( + `${stored} tokens stored, ${ownerGone.length} skipped, ${ownerDeleting.length} owners being deleted, ${heldByOther.length} held by another user`, + ); + + if (ownerDeleting.length || heldByOther.length) { + process.exitCode = 1; + } + } finally { + await close(); + } +}; + +main().catch((error: unknown) => { + console.error('ERROR:', error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/scripts/open-pool.ts b/scripts/open-pool.ts new file mode 100644 index 0000000..b1e3e51 --- /dev/null +++ b/scripts/open-pool.ts @@ -0,0 +1,37 @@ +import type { Pool } from 'pg'; + +import { + buildRuntimePool, + connectCloudSqlRuntime, + resolveRuntimeConnection, + setCloudSqlRuntimeStream, +} from '../db/connection.ts'; + +/** + * A one-connection pool for a maintenance script, on DATABASE_URL or on the + * WEBSITE_DB_* Cloud SQL variables the runtime uses, and the function that + * closes it. + */ +export const openPool = async (): Promise<{ pool: Pool; close: () => Promise }> => { + const connection = resolveRuntimeConnection(); + + if (connection.kind === 'url') { + const pool = buildRuntimePool(process.env, { max: 1 }); + + return { pool, close: () => pool.end() }; + } + + const { connector, stream } = await connectCloudSqlRuntime(connection); + + setCloudSqlRuntimeStream(stream); + + const pool = buildRuntimePool(process.env, { max: 1 }); + + return { + pool, + close: async () => { + await pool.end(); + connector.close(); + }, + }; +}; diff --git a/scripts/share-migration-round-trip.ts b/scripts/share-migration-round-trip.ts new file mode 100644 index 0000000..e8f896b --- /dev/null +++ b/scripts/share-migration-round-trip.ts @@ -0,0 +1,671 @@ +// The share migration end to end, on a local website and the dfo-apis local +// stack: seeds one share row per legacy case in the website database, runs +// export-shares, dfo-apis' import-shares and import-share-tokens, renders +// /s/ and /embed/ of each through the dev server, writes to the +// signed-in owner's share with its imported token, and runs both imports +// again to show they change nothing. docs/share-export.md says how to run it. +// +// node scripts/share-migration-round-trip.ts +// +// Reads DATABASE_URL (the dev server's database, migrated), BETTER_AUTH_SECRET +// (the dev server's, to sign the owner's session cookie), WEBSITE_URL (the dev +// server, default http://127.0.0.1:3000) and DFO_APIS_DIR (the dfo-apis +// checkout whose `just local-stack` is up and seeded). Each check prints one +// line, and the run exits 1 when any fails. It leaves its rows and shares in +// place, under ids no other run uses. +import { execFile } from 'node:child_process'; +import { createHmac, randomBytes, randomUUID } from 'node:crypto'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { promisify } from 'node:util'; + +import { openPool } from './open-pool.ts'; + +const run = promisify(execFile); + +const WEBSITE = (process.env.WEBSITE_URL ?? 'http://127.0.0.1:3000').replace(/\/$/, ''); +const APIS_DIR = process.env.DFO_APIS_DIR; +const AUTH_SECRET = process.env.BETTER_AUTH_SECRET; +// The address the site's front end would append for this script, a +// documentation address (RFC 5737). +const CLIENT = { 'x-forwarded-for': '203.0.113.11' }; +const SESSION_COOKIE = 'better-auth.session_token'; + +let failures = 0; + +const check = (name: string, ok: boolean, detail = ''): void => { + if (!ok) { + failures += 1; + } + + console.log(`${ok ? 'ok ' : 'FAIL'} ${name}${!ok && detail ? `: ${detail}` : ''}`); +}; + +const SHARE_ALPHABET = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + +// A fresh 8-character share id, so a run never meets an earlier run's shares. +const shareId = (): string => + Array.from(randomBytes(8), (byte) => SHARE_ALPHABET[byte % SHARE_ALPHABET.length]).join(''); + +type Reading = { name: string; spec_id: string; content_version: string }; +type CardRef = { component_id: string; revision: number }; +type Frame = { spec_id: string; content_version: string }; + +// A card in dfo-apis on `frame`, made through the site's proxy. +const makeCard = async (type: string, title: string, frame: Frame): Promise => { + const manifest = (await ( + await fetch(`${WEBSITE}/api/v1/datasets/${frame.spec_id}/${frame.content_version}`) + ).json()) as { definition: { identity: { shape: { id: string } } } }; + const response = await fetch(`${WEBSITE}/api/v1/presentation/components`, { + method: 'POST', + headers: { 'content-type': 'application/json', ...CLIENT }, + body: JSON.stringify({ + intent: { + type, + title, + frame_ref: { spec_id: frame.spec_id, content_version: frame.content_version }, + select: { stat_var: `dcid:${manifest.definition.identity.shape.id}` }, + }, + }), + }); + const body = (await response.json()) as CardRef & { code?: string }; + + if (!response.ok) { + throw new Error(`card create answered ${response.status} ${body.code}`); + } + + return { component_id: body.component_id, revision: body.revision }; +}; + +const chart = (id: string, turn: number, title: string, extra: Record = {}) => ({ + kind: 'chart', + id, + turn, + chartType: 'line', + title, + categories: ['2022', '2023'], + series: [{ label: 'Kenya', values: [1, 2] }], + ...extra, +}); + +const table = (id: string, turn: number, title: string, extra: Record = {}) => ({ + kind: 'table', + id, + turn, + title, + columns: ['Year', 'Value'], + rows: [['2023', '3.5']], + ...extra, +}); + +// A saved conversation as the website stored a share's copy of it. +const conversation = (title: string, widgets: unknown[], rest: Record = {}) => ({ + id: `conv-${randomUUID()}`, + sessionId: 'harness-session', + componentRefs: [], + title, + updatedAt: 1, + messages: [], + widgets, + activity: [], + turnCount: 1, + pinnedIds: [], + removedIds: [], + layout: {}, + textCards: [], + titles: {}, + notes: {}, + views: {}, + ...rest, +}); + +type Case = { + label: string; + id: string; + kind: 'search' | 'canvas' | 'card' | 'conversation'; + mode: 'live' | 'snapshot' | null; + owned: boolean; + title: string; + data: unknown; + parts: Record[]; + // Text each Studio page shows, server-rendered, and text it must not. + // A slotted card draws its component revision, and a live one is held + // for the browser, named by its component id. + shows: string[]; + hides: string[]; + // The titles a live share's slots draw, from their component revisions. + slots: string[]; +}; + +const searchPart = (index: number, reading: Reading, pinned: boolean) => ({ + part_index: index, + spec_id: reading.spec_id, + identity: { variable: reading.name }, + entity_groups: null, + variable_text: reading.name, + content_version: pinned ? reading.content_version : null, + artifact_digest: null, + operations: [], + assumptions: null, +}); + +const page = async (pathname: string) => { + const response = await fetch(`${WEBSITE}${pathname}`, { redirect: 'manual', headers: CLIENT }); + + return { + status: response.status, + location: response.headers.get('location'), + text: await response.text(), + }; +}; + +// The counts in import-shares' `presentation_shares_imported` line. +const importCounts = (output: string): { imported?: number; unchanged?: number } => + JSON.parse( + output.split('\n').find((line) => line.includes('presentation_shares_imported')) ?? '{}', + ); + +const main = async (): Promise => { + if (!APIS_DIR || !AUTH_SECRET || !process.env.DATABASE_URL) { + console.error( + 'set DATABASE_URL, BETTER_AUTH_SECRET and DFO_APIS_DIR; see docs/share-export.md', + ); + process.exitCode = 2; + + return; + } + + const readings = JSON.parse( + await fs.readFile(path.join(APIS_DIR, 'runs/local-stack/seeded.json'), 'utf8'), + ) as Reading[]; + const [oda, population] = ['oda_germany_to_kenya_ghana_nigeria', 'kenya_population'].map( + (name) => { + const reading = readings.find((r) => r.name === name); + + if (!reading) { + throw new Error(`the local stack seeded no ${name}`); + } + + return reading; + }, + ) as [Reading, Reading]; + + const odaCard = await makeCard('line_chart', 'German aid, round trip', oda); + const peopleCard = await makeCard('table', 'Kenya population, round trip', population); + const odaFrame = { spec_id: oda.spec_id, content_version: oda.content_version }; + + const cases: Case[] = [ + { + label: 'search, no mode, guest', + id: shareId(), + kind: 'search', + mode: null, + owned: false, + title: 'aid to Kenya', + data: { query: 'aid to Kenya' }, + parts: [searchPart(0, oda, false)], + shows: [], + hides: [], + slots: [], + }, + { + label: 'search snapshot, owner', + id: shareId(), + kind: 'search', + mode: 'snapshot', + owned: true, + title: 'people in Kenya', + data: { query: 'people in Kenya' }, + parts: [searchPart(0, population, true)], + shows: [], + hides: [], + slots: [], + }, + { + label: 'canvas, no mode, a card with no ref, owner', + id: shareId(), + kind: 'canvas', + mode: null, + owned: true, + title: 'Round trip canvas', + data: conversation( + 'Round trip canvas', + [ + chart('c1', 1, 'Stored chart title', { ref: odaCard, frame: odaFrame }), + table('c2', 1, 'Inline table with no ref'), + ], + { + textCards: [ + { kind: 'text', id: 't1', title: 'Note', body: 'Words from the owner' }, + ], + }, + ), + parts: [], + shows: ['German aid, round trip', 'Inline table with no ref', 'Words from the owner'], + hides: ['Stored chart title'], + slots: [], + }, + { + label: 'canvas live, a card with no frame, owner', + id: shareId(), + kind: 'canvas', + mode: 'live', + owned: true, + title: 'Round trip live canvas', + data: conversation('Round trip live canvas', [ + chart('c1', 1, 'Stored chart title', { ref: odaCard, frame: odaFrame }), + table('c2', 1, 'Stored table title', { ref: peopleCard }), + ]), + parts: [], + shows: [odaCard.component_id, peopleCard.component_id], + hides: ['Stored chart title', 'Stored table title'], + slots: ['German aid, round trip', 'Kenya population, round trip'], + }, + { + label: 'card live, owner', + id: shareId(), + kind: 'card', + mode: 'live', + owned: true, + title: 'Round trip live card', + data: conversation( + 'Round trip live card', + [chart('c1', 1, 'Stored chart title', { ref: odaCard, frame: odaFrame })], + { pinnedIds: ['c1'] }, + ), + parts: [], + shows: [odaCard.component_id], + hides: ['Stored chart title'], + slots: ['German aid, round trip'], + }, + { + label: 'card snapshot with no ref, guest', + id: shareId(), + kind: 'card', + mode: 'snapshot', + owned: false, + title: 'Guest card', + data: conversation('Guest card', [table('c1', 1, 'Guest table with no ref')], { + pinnedIds: ['c1'], + }), + parts: [], + shows: ['Guest table with no ref'], + hides: [], + slots: [], + }, + { + label: 'conversation, no mode, owner', + id: shareId(), + kind: 'conversation', + mode: null, + owned: true, + title: 'Round trip conversation', + data: conversation( + 'Round trip conversation', + [chart('c1', 1, 'Stored chart title', { ref: odaCard, frame: odaFrame })], + { + messages: [ + { role: 'user', text: 'How much aid did Germany give Kenya?' }, + { role: 'bot', text: 'Here is German aid to Kenya.' }, + ], + }, + ), + parts: [], + // A conversation keeps its stored copy whole, cards included. + shows: [ + 'How much aid did Germany give Kenya?', + 'Here is German aid to Kenya.', + 'Stored chart title', + ], + hides: [], + slots: [], + }, + ]; + + const work = await fs.mkdtemp(path.join(os.tmpdir(), 'share-round-trip-')); + const exported = path.join(work, 'shares.jsonl'); + const tokens = path.join(work, 'share-tokens.jsonl'); + const { pool, close } = await openPool(); + + try { + const owner = `owner-${randomUUID()}`; + // The owner's share_token rows, as share id to token. + const ownerTokens = async () => + JSON.stringify( + ( + await pool.query( + 'select share_id, write_token from share_token where user_id = $1 order by share_id collate "C"', + [owner], + ) + ).rows.map((row: { share_id: string; write_token: string }) => [ + row.share_id, + row.write_token, + ]), + ); + // Every case is made after this moment, in case order, so the export + // takes exactly them. + const since = new Date(Date.now() - 1000); + + await pool.query( + `insert into "user" (id, name, email, email_verified) values ($1, 'Owner', $2, true)`, + [owner, `${owner}@example.org`], + ); + + for (const [index, share] of cases.entries()) { + await pool.query( + `insert into share (id, user_id, kind, mode, title, data, created_at) + values ($1, $2, $3, $4, $5, $6::jsonb, $7)`, + [ + share.id, + share.owned ? owner : null, + share.kind, + share.mode, + share.title, + JSON.stringify(share.data), + new Date(since.getTime() + 1 + index), + ], + ); + + for (const part of share.parts) { + await pool.query( + `insert into share_ref (share_id, part_index, spec_id, identity, entity_groups, + variable_text, content_version, artifact_digest, operations, assumptions) + values ($1, $2, $3, $4::jsonb, $5::jsonb, $6, $7, $8, $9::jsonb, $10::jsonb)`, + [ + share.id, + part.part_index, + part.spec_id, + JSON.stringify(part.identity), + JSON.stringify(part.entity_groups), + part.variable_text, + part.content_version, + part.artifact_digest, + JSON.stringify(part.operations), + JSON.stringify(part.assumptions), + ], + ); + } + } + + console.log(`seeded ${cases.length} shares since ${since.toISOString()}:`); + + for (const share of cases) { + console.log(` ${share.id} ${share.label}`); + } + + const before = await page(`/s/${cases[2]!.id}`); + + check( + 'a share the website alone holds is not found before the import', + before.status === 404, + `${before.status}`, + ); + + const migrate = async () => { + const out = await run( + process.execPath, + ['scripts/export-shares.ts', '--since', since.toISOString(), '--out', exported], + { env: process.env }, + ); + const imported = await run( + 'uv', + [ + 'run', + '--no-sync', + '--env-file', + 'scripts/local-stack/local.env', + 'datasets-maintenance', + 'import-shares', + '--in', + exported, + '--tokens', + tokens, + ], + { cwd: APIS_DIR, env: { ...process.env, PYTHONPATH: APIS_DIR } }, + ); + const stored = await run( + process.execPath, + ['scripts/import-share-tokens.ts', '--in', tokens], + { env: process.env }, + ); + + return { + export: out.stderr.trim(), + import: (imported.stdout + imported.stderr).trim(), + tokens: stored.stderr.trim(), + }; + }; + + const first = await migrate(); + + console.log(`export-shares: ${first.export}`); + console.log(`import-shares: ${first.import}`); + console.log(`import-share-tokens: ${first.tokens}`); + + const lines = (await fs.readFile(exported, 'utf8')) + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line) as { id: string }); + + check( + 'the export holds every seeded share, in order', + JSON.stringify(lines.map(({ id }) => id)) === JSON.stringify(cases.map(({ id }) => id)), + lines.map(({ id }) => id).join(' '), + ); + check( + 'import-shares imported every share', + importCounts(first.import).imported === cases.length, + first.import, + ); + + const tokenLines = (await fs.readFile(tokens, 'utf8')) + .trimEnd() + .split('\n') + .map((line) => JSON.parse(line) as { id: string; owner: string; write_token: string }); + const owned = cases.filter(({ owned }) => owned).map(({ id }) => id); + + check( + "a token for each signed-in owner's share and none for a guest's", + JSON.stringify(tokenLines.map(({ id }) => id).sort()) === + JSON.stringify([...owned].sort()) && + tokenLines.every((line) => line.owner === owner), + tokenLines.map(({ id }) => id).join(' '), + ); + + const reported = JSON.stringify( + tokenLines + .map((line) => [line.id, line.write_token]) + .sort(([a = ''], [b = '']) => (a < b ? -1 : 1)), + ); + const kept = await ownerTokens(); + + check('import-share-tokens kept each token for the owner', kept === reported, kept); + + for (const share of cases) { + const shown = await page(`/s/${share.id}`); + const embedded = await page(`/embed/${share.id}`); + + if (share.kind === 'search') { + check( + `${share.id} ${share.label}: /s redirects to its search`, + shown.status === 307 && shown.location === `/data?share=${share.id}`, + `${shown.status} ${shown.location}`, + ); + check( + `${share.id} ${share.label}: /embed is not found`, + embedded.status === 404, + `${embedded.status}`, + ); + + const read = await fetch(`${WEBSITE}/api/share/${share.id}`); + const body = (await read.json()) as { + mode?: string; + query?: string; + refs?: unknown[]; + }; + const expectedMode = share.mode ?? 'live'; + + check( + `${share.id} ${share.label}: its references read back as ${expectedMode}`, + read.status === 200 && + body.mode === expectedMode && + body.query === share.title && + body.refs?.length === share.parts.length, + `${read.status} ${JSON.stringify(body).slice(0, 300)}`, + ); + continue; + } + + for (const [name, answer] of [ + ['/s', shown], + ['/embed', embedded], + ] as const) { + const missing = share.shows.filter((text) => !answer.text.includes(text)); + const extra = share.hides.filter((text) => answer.text.includes(text)); + + check( + `${share.id} ${share.label}: ${name} answers 200 with its cards`, + answer.status === 200 && missing.length === 0 && extra.length === 0, + `${answer.status}, missing ${JSON.stringify(missing)}, shown ${JSON.stringify(extra)}`, + ); + } + + for (const [index, title] of share.slots.entries()) { + const slot = await fetch( + `${WEBSITE}/api/v1/presentation/shares/${share.id}/slots/${index + 1}`, + { headers: CLIENT }, + ); + const text = await slot.text(); + + check( + `${share.id} ${share.label}: live slot ${index + 1} draws "${title}"`, + slot.status === 200 && text.includes(title), + `${slot.status} ${text.slice(0, 300)}`, + ); + } + } + + // The owner signs in with a session row and its cookie, signed as + // Better Auth signs it. + const token = randomBytes(24).toString('base64url'); + + await pool.query( + `insert into session (id, token, user_id, expires_at, created_at, updated_at) + values ($1, $2, $3, now() + interval '1 hour', now(), now())`, + [randomUUID(), token, owner], + ); + + const signature = createHmac('sha256', AUTH_SECRET).update(token).digest('base64'); + const cookie = `${SESSION_COOKIE}=${encodeURIComponent(`${token}.${signature}`)}`; + const live = cases.find(({ kind, mode }) => kind === 'canvas' && mode === 'live')!; + const guest = cases.find(({ owned, kind }) => !owned && kind === 'card')!; + const write = (id: string, method: string, withCookie = true) => + fetch(`${WEBSITE}/api/share/${id}`, { + method, + headers: { + 'content-type': 'application/json', + ...CLIENT, + ...(withCookie ? { cookie } : {}), + }, + ...(method === 'PATCH' ? { body: JSON.stringify({ mode: 'snapshot' }) } : {}), + }); + + const anonymous = await write(live.id, 'PATCH', false); + + check( + 'a write without a session is refused', + anonymous.status === 401, + `${anonymous.status}`, + ); + + const switched = await write(live.id, 'PATCH'); + const switchedBody = await switched.text(); + + check( + `${live.id} the owner switches their imported live canvas to snapshot`, + switched.status === 200 && JSON.parse(switchedBody).mode === 'snapshot', + `${switched.status} ${switchedBody}`, + ); + + const record = await fetch(`${WEBSITE}/api/v1/presentation/shares/${live.id}`, { + headers: CLIENT, + }); + const recordBody = (await record.json()) as { mode?: string }; + + check( + `${live.id} dfo-apis holds it as a snapshot`, + recordBody.mode === 'snapshot', + JSON.stringify(recordBody).slice(0, 300), + ); + + const refused = await write(guest.id, 'DELETE'); + + check( + `${guest.id} the owner cannot delete a guest's share`, + refused.status === 404, + `${refused.status}`, + ); + check( + `${guest.id} the guest's share still renders`, + (await page(`/s/${guest.id}`)).status === 200, + ); + + // Both imports again change nothing in dfo-apis or the website. The + // reads go one at a time, since the shares door answers a retryable + // 503 past four requests in flight. + const readAll = async () => { + const records = []; + + for (const { id } of cases) { + const response = await fetch(`${WEBSITE}/api/v1/presentation/shares/${id}`, { + headers: CLIENT, + }); + + records.push(await response.json()); + } + + return JSON.stringify(records); + }; + const recordsBefore = await readAll(); + const tokensBefore = await ownerTokens(); + const reportBefore = await fs.readFile(tokens, 'utf8'); + const again = await migrate(); + + console.log(`export-shares again: ${again.export}`); + console.log(`import-shares again: ${again.import}`); + console.log(`import-share-tokens again: ${again.tokens}`); + check( + 'a second import-shares imports none and finds every share imported', + importCounts(again.import).imported === 0 && + importCounts(again.import).unchanged === cases.length, + again.import, + ); + check( + 'a second import-shares writes no token line', + (await fs.readFile(tokens, 'utf8')) === reportBefore, + ); + const recordsAfter = await readAll(); + + check( + 'dfo-apis holds the same shares after both imports again', + recordsAfter === recordsBefore, + `before ${recordsBefore}\nafter ${recordsAfter}`, + ); + check( + 'share_token holds the same tokens after both imports again', + (await ownerTokens()) === tokensBefore, + ); + } finally { + await close(); + await fs.rm(work, { recursive: true, force: true }); + } + + console.log(failures ? `${failures} checks failed` : 'every check passed'); + process.exitCode = failures ? 1 : 0; +}; + +main().catch((error: unknown) => { + console.error('ERROR:', error instanceof Error ? error.stack : error); + process.exitCode = 1; +}); diff --git a/src/app/api/presentation/components/[id]/revisions/[revision]/data/route.ts b/src/app/api/presentation/components/[id]/revisions/[revision]/data/route.ts index f82d204..2fa07e3 100644 --- a/src/app/api/presentation/components/[id]/revisions/[revision]/data/route.ts +++ b/src/app/api/presentation/components/[id]/revisions/[revision]/data/route.ts @@ -1,6 +1,8 @@ import { NextRequest } from 'next/server'; import { forward } from '@/lib/proxy/forward'; +import { dotSegmentProblem } from '@/lib/proxy/problem'; +import { isForwardablePath } from '@/lib/proxy/surfaces'; import { searchApiBase } from '@/lib/searchApi'; import { componentDataTarget } from '@/lib/studio/cardDownload'; @@ -15,6 +17,12 @@ export const GET = async ( ) => { const { id, revision } = await params; + // encodeURIComponent leaves `.` and `..` as they are, so a bare dot id + // would climb out of the components path once fetch resolves the URL. + if (!isForwardablePath([id, revision])) { + return dotSegmentProblem(); + } + return forward(req, { surface: 'api', target: componentDataTarget(searchApiBase(), id, revision, req.nextUrl.search), diff --git a/src/app/api/presentation/components/[id]/route.ts b/src/app/api/presentation/components/[id]/route.ts index 0bc0448..23ee8f5 100644 --- a/src/app/api/presentation/components/[id]/route.ts +++ b/src/app/api/presentation/components/[id]/route.ts @@ -1,20 +1,26 @@ import { NextRequest } from 'next/server'; import { forward } from '@/lib/proxy/forward'; +import { dotSegmentProblem } from '@/lib/proxy/problem'; +import { forwardablePath } from '@/lib/proxy/surfaces'; import { searchApiBase } from '@/lib/searchApi'; // The Presentation API's component read (GET /api/presentation/components/ // {id}?revision=N: one revision's stored intent, fact summary and resolved // payload), mirrored under the site's domain — see src/lib/proxy/forward. // Studio reads a revision here when its `component.revised` frame carries -// no payload. Only the read is exposed; creating and revising stay with the -// harness. +// no payload. This route only reads: cards are created and revised through +// /api/v1/presentation/components, and dfo-apis bounds them per caller. export const GET = async (req: NextRequest, { params }: { params: Promise<{ id: string }> }) => { - const { id } = await params; + const rest = forwardablePath([(await params).id]); + + if (rest === undefined) { + return dotSegmentProblem(); + } return forward(req, { surface: 'api', - target: `${searchApiBase()}/api/presentation/components/${encodeURIComponent(id)}${req.nextUrl.search}`, + target: `${searchApiBase()}/api/presentation/components${rest}${req.nextUrl.search}`, unreachable: 'The Presentation API is unreachable.', }); }; diff --git a/src/app/api/share/[id]/cards/[cardId]/data/route.ts b/src/app/api/share/[id]/cards/[cardId]/data/route.ts index 8c96ae9..35f133b 100644 --- a/src/app/api/share/[id]/cards/[cardId]/data/route.ts +++ b/src/app/api/share/[id]/cards/[cardId]/data/route.ts @@ -1,21 +1,16 @@ import type { NextRequest } from 'next/server'; -import { apisFetch } from '@/lib/apisAuth'; -import { db } from '@/lib/db'; -import { dataResponse } from '@/lib/studio/liveShare'; +import { slotDepsFor } from '@/lib/chat/shareStore'; +import { slotDataResponse } from '@/lib/studio/slotCards'; type Context = { params: Promise<{ id: string; cardId: string }> }; -// GET /api/share//cards//data?format=&bundle= → a file -// The rows behind a live share's card as its last refresh drew it, streamed -// from dfo-apis. Public, like the link itself. See src/lib/studio/liveShare. +// GET /api/share//cards//data?slot=&format=&bundle= → a file +// The rows behind the card slot n of a share draws, on the frame the slot +// serves now, streamed from dfo-apis. Public, like the link itself; 404 +// without a slot. See src/lib/studio/slotCards. export const GET = async (request: NextRequest, { params }: Context) => { - const { id, cardId } = await params; + const { id } = await params; - return dataResponse( - { db, env: process.env, fetch: apisFetch }, - id, - cardId, - request.nextUrl.searchParams, - ); + return slotDataResponse(slotDepsFor(request.headers), id, request.nextUrl.searchParams); }; diff --git a/src/app/api/share/[id]/cards/[cardId]/refresh/route.ts b/src/app/api/share/[id]/cards/[cardId]/refresh/route.ts index 0e517b2..84c51fc 100644 --- a/src/app/api/share/[id]/cards/[cardId]/refresh/route.ts +++ b/src/app/api/share/[id]/cards/[cardId]/refresh/route.ts @@ -1,16 +1,27 @@ -import { apisFetch } from '@/lib/apisAuth'; -import { db } from '@/lib/db'; -import { refreshResponse } from '@/lib/studio/liveShare'; +import type { NextRequest } from 'next/server'; + +import { slotDepsFor } from '@/lib/chat/shareStore'; +import { slotOfQuery, slotRefreshResponse } from '@/lib/studio/slotCards'; type Context = { params: Promise<{ id: string; cardId: string }> }; -// POST /api/share//cards//refresh → { status, card?, retry_after_s?, code? } -// One data card of a live canvas or card share on current data. Public, like -// the link itself. The server calls dfo-apis or dfo-ai with a credential -// that stays here. The route sits outside the /api/studio proxy, which -// forwards any path. See src/lib/studio/liveShare. -export const POST = async (_request: Request, { params }: Context) => { - const { id, cardId } = await params; +// GET /api/share//cards//refresh?slot= → { status, card?, code?, retryable? } +// One data card of a live canvas or card share, drawn on the frame slot n +// of the share in dfo-apis serves now. Public, like the link itself. The +// share page names each card's slot, so the route reads that slot alone; +// without one it answers 404. dfo-apis' ETag goes back with it and the +// browser's If-None-Match goes on to dfo-apis, so an unchanged card answers +// 304. POST answers the same, for pages that send it. See +// src/lib/studio/slotCards. +export const GET = async (request: NextRequest, { params }: Context) => { + const { id } = await params; - return refreshResponse({ db, env: process.env, fetch: apisFetch }, id, cardId); + return slotRefreshResponse( + slotDepsFor(request.headers), + id, + slotOfQuery(request.nextUrl.searchParams.get('slot')), + request.headers.get('if-none-match'), + ); }; + +export const POST = GET; diff --git a/src/app/api/share/[id]/route.ts b/src/app/api/share/[id]/route.ts index 6c739c3..72ce1d3 100644 --- a/src/app/api/share/[id]/route.ts +++ b/src/app/api/share/[id]/route.ts @@ -1,8 +1,9 @@ import { NextResponse } from 'next/server'; import { getUser } from '@/lib/auth/session'; -import { deleteShare, snapshotShare } from '@/lib/chat/shareStore'; -import type { OwnerWriteResult } from '@/lib/chat/shareWrites'; +import { SHARE_ID, SharesError } from '@/lib/chat/apisShares'; +import { sharesReadFailure } from '@/lib/chat/shareRefusal'; +import { writeOwnShare } from '@/lib/chat/shareStore'; import { missingProblem, problem, @@ -13,16 +14,25 @@ import { getSearchShare } from '@/lib/search/searchShare'; type Context = { params: Promise<{ id: string }> }; -const SHARE_ID = /^[A-Za-z0-9]{8}$/; - const noSuchShare = (): Response => missingProblem('There is no such share.'); // GET /api/share/ → a search share's references ({ mode, query, refs }). -// Public, like the link itself; conversation shares render server-side on +// Public, like the link itself; Studio shares render server-side on // /s/ and never come through here. +// A failed read of dfo-apis answers a problem document with its code. export const GET = async (_request: Request, { params }: Context) => { const { id } = await params; - const found = SHARE_ID.test(id) ? await getSearchShare(id) : null; + let found: Awaited> = null; + + try { + found = SHARE_ID.test(id) ? await getSearchShare(id) : null; + } catch (error) { + if (error instanceof SharesError) { + return sharesReadFailure(error); + } + + throw error; + } if (!found) { return noSuchShare(); @@ -31,12 +41,14 @@ export const GET = async (_request: Request, { params }: Context) => { return NextResponse.json(found, { headers: { 'Cache-Control': 'no-store' } }); }; -// An owner's write to their own Studio share. A share that does not exist -// and one that belongs to someone else both answer 404, so the answer does +// An owner's write to their own share in dfo-apis, with the write token the +// site kept when they made it. A share that does not exist, one the caller +// holds no token for, and a guest's share all answer 404, so the answer does // not reveal which ids are taken. const ownerWrite = async ( + request: Request, { params }: Context, - write: (id: string, userId: string) => Promise, + action: 'snapshot' | 'delete', done: (id: string) => Response, ) => { const user = await getUser(); @@ -46,27 +58,49 @@ const ownerWrite = async ( } const { id } = await params; - const result = SHARE_ID.test(id) ? await write(id, user.id) : 'not_found'; - if (result === 'not_found') { - return noSuchShare(); - } + try { + const result = await writeOwnShare(request.headers, { + shareId: id, + userId: user.id, + action, + }); + + if (result === 'unsupported_kind') { + return problem({ + status: 400, + code: 'unsupported_kind', + detail: 'Only a canvas or card share can be switched to a snapshot.', + retryable: false, + }); + } + + return result === 'done' ? done(id) : noSuchShare(); + } catch (error) { + if (!(error instanceof SharesError)) { + throw error; + } + + console.error('share: dfo-apis refused an owner write', { + action, + code: error.code, + status: error.status, + }); - if (result === 'unsupported_kind') { return problem({ - status: 400, - code: 'unsupported_kind', - detail: 'Only a conversation, canvas, or card share can be changed here.', - retryable: false, + status: 502, + code: 'transient_unavailable', + detail: 'The share could not be changed right now. Try again in a minute.', + retryable: true, }); } - - return done(id); }; // PATCH /api/share/ { mode: 'snapshot' } → { id, mode } -// Switches the caller's live canvas or card share to snapshot. A snapshot -// cannot be switched to live here. +// Switches the caller's live canvas or card share to snapshot under the same +// URLs: each slot keeps the data it serves now. A snapshot answers as it is. +// A snapshot cannot be switched to live. A search or conversation share +// answers 400 `unsupported_kind`. export const PATCH = async (request: Request, context: Context) => { const body = (await request.json().catch(() => null)) as { mode?: unknown } | null; @@ -74,10 +108,12 @@ export const PATCH = async (request: Request, context: Context) => { return requestInvalidProblem('A share can only be switched to `snapshot`.', '/mode'); } - return ownerWrite(context, snapshotShare, (id) => NextResponse.json({ id, mode: 'snapshot' })); + return ownerWrite(request, context, 'snapshot', (id) => + NextResponse.json({ id, mode: 'snapshot' }), + ); }; // DELETE /api/share/ → 204 -// Removes the caller's conversation, canvas, or card share. -export const DELETE = async (_request: Request, context: Context) => - ownerWrite(context, deleteShare, () => new NextResponse(null, { status: 204 })); +// Takes down the caller's share, of any kind. +export const DELETE = async (request: Request, context: Context) => + ownerWrite(request, context, 'delete', () => new NextResponse(null, { status: 204 })); diff --git a/src/app/api/share/route.ts b/src/app/api/share/route.ts index 48e4a52..3e6995b 100644 --- a/src/app/api/share/route.ts +++ b/src/app/api/share/route.ts @@ -2,22 +2,25 @@ import { after, NextResponse, type NextRequest } from 'next/server'; import { getUser } from '@/lib/auth/session'; -import { GUEST_SHARES, readStudioShareRequest, shareCreateProps } from '@/lib/chat/shareMode'; -import { createShare, liveShareRefusalFor, spendGuestShare } from '@/lib/chat/shareStore'; -import { clientIp } from '@/lib/proxy/forward'; +import { SharesError } from '@/lib/chat/apisShares'; +import { sharesRefusal } from '@/lib/chat/shareRefusal'; +import { studioShareBody } from '@/lib/chat/sharePayload'; +import { readStudioShareRequest, shareCreateProps } from '@/lib/chat/shareMode'; +import { makeShare, slotDepsFor } from '@/lib/chat/shareStore'; import { readBoundedText } from '@/lib/proxy/boundedBody'; import { problem, requestInvalidProblem } from '@/lib/proxy/problem'; import { track } from '@/lib/events/track'; import { createSearchShare } from '@/lib/search/searchShare'; import { readSearchShareRequest } from '@/lib/search/searchShareRequest'; +import { liveCardsRefusal } from '@/lib/studio/slotCards'; // The largest share request read, in bytes; a larger one gets 413. const MAX_SHARE_BODY = 10_000_000; // POST /api/share { kind: 'search', mode, query, parts } → { id } // A search result's link: dataset references only, no numbers. The search -// page is public, so an account is optional here (the share is simply -// unowned without one). +// page is public, so an account is optional here. A signed-in sharer's +// write token is kept. A guest's share stays as made. const shareSearch = async (body: any, headers: Headers) => { const request = readSearchShareRequest(body); @@ -31,12 +34,17 @@ const shareSearch = async (body: any, headers: Headers) => { try { created = await createSearchShare( + headers, user?.id ?? null, request.mode, request.query, request.parts, ); } catch (error) { + if (error instanceof SharesError) { + return sharesRefusal(error); + } + // The platform refused the register/mint — say which, don't store a // link that can't open. The log names the datasets and the cause. const { message, cause } = error as Error; @@ -71,14 +79,18 @@ const shareSearch = async (body: any, headers: Headers) => { }; // POST /api/share { kind, conversation, cardId?, mode? } → { id, title } -// A pruned copy of what the caller is looking at, under a short public id. -// An account is optional. A guest's share has no owner, so nobody can switch -// or remove it: a guest gets snapshots only (403 `live_needs_account`), and -// GUEST_SHARES a day per network address (429 `guest_limit`). +// A pruned copy of what the caller is looking at, made in dfo-apis +// (src/lib/chat/sharePayload, src/lib/chat/apisShares) under a short public +// id. An account is optional. A signed-in sharer's write token is kept, so +// they can switch the share to a snapshot or remove it later. A guest's +// share, live or snapshot, stays as made. dfo-apis counts shares per +// network address (429 `share_limit`). // `mode` is 'live' or 'snapshot' for a canvas or a card and 'snapshot' for a // conversation. A request that names none stores a snapshot. A live share is -// refused when its data cards cannot all be refreshed (`liveShareRefusal` in -// src/lib/studio/liveShare), and the refusal names the cards in the way. +// checked before it is made (`liveCardsRefusal`), so none is left behind +// when its cards fail: 400 `live_cards_blocked` names a data card it shows +// that names no component revision, or one that cannot follow newer data, +// and 400 `live_too_many_cards` gives the cap. // Every refusal is a problem document (src/lib/proxy/problem). export const POST = async (request: NextRequest) => { const text = await readBoundedText(request.body, MAX_SHARE_BODY).catch(() => ''); @@ -110,35 +122,44 @@ export const POST = async (request: NextRequest) => { return read; } - const user = await getUser(); + const share = studioShareBody(read); - if (!user) { - if (read.mode === 'live') { - return problem({ - status: 403, - code: 'live_needs_account', - detail: 'Sign in to make a live share. Without an account, a share is a snapshot.', - retryable: false, - }); - } + if ('unslotted' in share) { + return problem({ + status: 400, + code: 'live_cards_blocked', + detail: 'Some data cards cannot show live data, so the share cannot be live.', + retryable: false, + extra: { cards: share.unslotted.map((id) => ({ id, reason: 'missing_ref' })) }, + }); + } - if (!(await spendGuestShare(clientIp(request)))) { - return problem({ - status: 429, - code: 'guest_limit', - detail: `This network has made today's ${GUEST_SHARES} guest shares. Sign in to make more.`, - retryable: false, - }); - } + // A card saved before cards kept their revision has nothing dfo-apis + // can draw, and a card share takes at least one card. + if (share.kind === 'card' && share.cards.length === 0) { + return requestInvalidProblem('This card has no saved revision, so it cannot be shared.'); } - const refusal = read.mode === 'live' ? await liveShareRefusalFor(read) : null; + const refusal = share.latest + ? await liveCardsRefusal(slotDepsFor(request.headers), share) + : null; if (refusal) { return refusal; } - const created = await createShare({ userId: user?.id ?? null, ...read }); + const user = await getUser(); + let created: { id: string }; + + try { + created = await makeShare(request.headers, user?.id ?? null, share); + } catch (error) { + if (error instanceof SharesError) { + return sharesRefusal(error); + } + + throw error; + } // Recorded after the response, so the link does not wait on analytics. after(() => @@ -149,5 +170,5 @@ export const POST = async (request: NextRequest) => { ), ); - return NextResponse.json(created); + return NextResponse.json({ id: created.id, title: share.title }); }; diff --git a/src/app/api/v1/datasets/[...path]/route.ts b/src/app/api/v1/datasets/[...path]/route.ts index aafb638..74101da 100644 --- a/src/app/api/v1/datasets/[...path]/route.ts +++ b/src/app/api/v1/datasets/[...path]/route.ts @@ -1,18 +1,26 @@ import { NextRequest } from 'next/server'; import { forward } from '@/lib/proxy/forward'; +import { dotSegmentProblem } from '@/lib/proxy/problem'; +import { forwardablePath } from '@/lib/proxy/surfaces'; import { searchApiBase } from '@/lib/searchApi'; // The Datasets API's document routes, mirrored 1:1 under the site's domain // (see src/lib/proxy/forward); the mint door itself is ../route.ts. const proxy = (req: NextRequest, { params }: { params: Promise<{ path: string[] }> }) => - params.then(({ path }) => - forward(req, { + params.then(({ path }) => { + const rest = forwardablePath(path); + + if (rest === undefined) { + return dotSegmentProblem(); + } + + return forward(req, { surface: 'api', - target: `${searchApiBase()}/api/datasets/${path.join('/')}${req.nextUrl.search}`, + target: `${searchApiBase()}/api/datasets${rest}${req.nextUrl.search}`, unreachable: 'The Datasets API is unreachable.', - }), - ); + }); + }); export const GET = proxy; export const POST = proxy; diff --git a/src/app/api/v1/graph/[...path]/route.ts b/src/app/api/v1/graph/[...path]/route.ts index 60af247..ea0c3e4 100644 --- a/src/app/api/v1/graph/[...path]/route.ts +++ b/src/app/api/v1/graph/[...path]/route.ts @@ -1,18 +1,26 @@ import { NextRequest } from 'next/server'; import { forward } from '@/lib/proxy/forward'; +import { dotSegmentProblem } from '@/lib/proxy/problem'; +import { forwardablePath } from '@/lib/proxy/surfaces'; import { searchApiBase } from '@/lib/searchApi'; // The Graph API (node, documentation, sources, resolve, capabilities), // mirrored 1:1 under the site's domain — see src/lib/proxy/forward. const proxy = (req: NextRequest, { params }: { params: Promise<{ path: string[] }> }) => - params.then(({ path }) => - forward(req, { + params.then(({ path }) => { + const rest = forwardablePath(path); + + if (rest === undefined) { + return dotSegmentProblem(); + } + + return forward(req, { surface: 'api', - target: `${searchApiBase()}/api/graph/${path.join('/')}${req.nextUrl.search}`, + target: `${searchApiBase()}/api/graph${rest}${req.nextUrl.search}`, unreachable: 'The Graph API is unreachable.', - }), - ); + }); + }); export const GET = proxy; export const POST = proxy; diff --git a/src/app/api/v1/presentation/components/[[...path]]/route.ts b/src/app/api/v1/presentation/components/[[...path]]/route.ts new file mode 100644 index 0000000..bda6405 --- /dev/null +++ b/src/app/api/v1/presentation/components/[[...path]]/route.ts @@ -0,0 +1,33 @@ +import { NextRequest } from 'next/server'; + +import { forward } from '@/lib/proxy/forward'; +import { PRESENTATION_HEADERS } from '@/lib/proxy/headers'; +import { dotSegmentProblem } from '@/lib/proxy/problem'; +import { forwardablePath } from '@/lib/proxy/surfaces'; +import { searchApiBase } from '@/lib/searchApi'; + +// The Presentation API's cards (create, revise, read, and a revision's data), +// mirrored 1:1 under the site's domain (see src/lib/proxy/forward). The write +// token a card was made with goes upstream (PRESENTATION_HEADERS). +const proxy = (req: NextRequest, { params }: { params: Promise<{ path?: string[] }> }) => + params.then(({ path = [] }) => { + const rest = forwardablePath(path); + + if (rest === undefined) { + return dotSegmentProblem(); + } + + return forward(req, { + surface: 'api', + target: `${searchApiBase()}/api/presentation/components${rest}${req.nextUrl.search}`, + pass: PRESENTATION_HEADERS, + unreachable: 'The Presentation API is unreachable.', + }); + }); + +export const GET = proxy; +export const POST = proxy; +export const PUT = proxy; +export const PATCH = proxy; +export const DELETE = proxy; +export const OPTIONS = proxy; diff --git a/src/app/api/v1/presentation/shares/[[...path]]/route.ts b/src/app/api/v1/presentation/shares/[[...path]]/route.ts new file mode 100644 index 0000000..19e4ce0 --- /dev/null +++ b/src/app/api/v1/presentation/shares/[[...path]]/route.ts @@ -0,0 +1,35 @@ +import { NextRequest } from 'next/server'; + +import { forward } from '@/lib/proxy/forward'; +import { PRESENTATION_HEADERS } from '@/lib/proxy/headers'; +import { dotSegmentProblem } from '@/lib/proxy/problem'; +import { forwardablePath } from '@/lib/proxy/surfaces'; +import { searchApiBase } from '@/lib/searchApi'; + +// The Presentation API's shares (create, replace, delete, read, a slot's +// data at /{id}/slots/{n}/data, and /{id}/history), mirrored 1:1 under the +// site's domain (see src/lib/proxy/forward). The write token, the slot data's +// ETag and If-None-Match, and CORS for pages on other origins pass through +// (PRESENTATION_HEADERS). +const proxy = (req: NextRequest, { params }: { params: Promise<{ path?: string[] }> }) => + params.then(({ path = [] }) => { + const rest = forwardablePath(path); + + if (rest === undefined) { + return dotSegmentProblem(); + } + + return forward(req, { + surface: 'api', + target: `${searchApiBase()}/api/presentation/shares${rest}${req.nextUrl.search}`, + pass: PRESENTATION_HEADERS, + unreachable: 'The Presentation API is unreachable.', + }); + }); + +export const GET = proxy; +export const POST = proxy; +export const PUT = proxy; +export const PATCH = proxy; +export const DELETE = proxy; +export const OPTIONS = proxy; diff --git a/src/app/api/v1/search/[...path]/route.ts b/src/app/api/v1/search/[...path]/route.ts index 0f2761f..971ada5 100644 --- a/src/app/api/v1/search/[...path]/route.ts +++ b/src/app/api/v1/search/[...path]/route.ts @@ -1,19 +1,27 @@ import { NextRequest } from 'next/server'; import { forward } from '@/lib/proxy/forward'; +import { dotSegmentProblem } from '@/lib/proxy/problem'; +import { forwardablePath } from '@/lib/proxy/surfaces'; import { searchApiBase } from '@/lib/searchApi'; // The Search API, mirrored 1:1 under the site's domain (see // src/lib/proxy/forward). Every door the engine has, every method — // resolve streams (SSE) and passes through unbuffered. const proxy = (req: NextRequest, { params }: { params: Promise<{ path: string[] }> }) => - params.then(({ path }) => - forward(req, { + params.then(({ path }) => { + const rest = forwardablePath(path); + + if (rest === undefined) { + return dotSegmentProblem(); + } + + return forward(req, { surface: 'api', - target: `${searchApiBase()}/api/search/${path.join('/')}${req.nextUrl.search}`, + target: `${searchApiBase()}/api/search${rest}${req.nextUrl.search}`, unreachable: 'The Search API is unreachable.', - }), - ); + }); + }); export const GET = proxy; export const POST = proxy; diff --git a/src/app/embed/[id]/page.tsx b/src/app/embed/[id]/page.tsx index 9ba81ca..f8083ab 100644 --- a/src/app/embed/[id]/page.tsx +++ b/src/app/embed/[id]/page.tsx @@ -4,10 +4,10 @@ import { notFound } from 'next/navigation'; import { after } from 'next/server'; import EmbedView from '@/components/organisms/EmbedView/EmbedView'; -import { getShare } from '@/lib/chat/shareStore'; +import { getShare, slotDepsFor } from '@/lib/chat/shareStore'; import { refererHost } from '@/lib/events/referer'; import { track } from '@/lib/events/track'; -import { sharedViewOf } from '@/lib/studio/liveCards'; +import { sharedViewOfRecord } from '@/lib/studio/slotCards'; type EmbedPageProps = { params: Promise<{ id: string }> }; @@ -17,14 +17,15 @@ export const generateMetadata = async ({ params }: EmbedPageProps): Promise { const { id } = await params; const row = await getShare(id); @@ -40,7 +41,7 @@ const EmbedPage = async ({ params }: EmbedPageProps) => { const host = refererHost(incoming.get('referer')); after(() => track('embed.view', { shareId: id, kind: row.kind, host }, { headers: incoming })); - return ; + return ; }; export default EmbedPage; diff --git a/src/app/s/[id]/page.tsx b/src/app/s/[id]/page.tsx index 9ef5e96..3b1af2f 100644 --- a/src/app/s/[id]/page.tsx +++ b/src/app/s/[id]/page.tsx @@ -4,11 +4,10 @@ import { notFound, redirect } from 'next/navigation'; import { after } from 'next/server'; import Studio from '@/components/organisms/Studio/Studio'; -import { isLiveStudioShare, searchShareMode, type ShareMode } from '@/lib/chat/shareMode'; -import { getShare } from '@/lib/chat/shareStore'; +import { getShare, slotDepsFor } from '@/lib/chat/shareStore'; import { refererHost } from '@/lib/events/referer'; import { track } from '@/lib/events/track'; -import { sharedViewOf } from '@/lib/studio/liveCards'; +import { sharedViewOfRecord } from '@/lib/studio/slotCards'; type SharePageProps = { params: Promise<{ id: string }> }; @@ -20,24 +19,15 @@ export const generateMetadata = async ({ params }: SharePageProps): Promise { - if (row.kind === 'search') { - return searchShareMode(row.mode); - } - - return isLiveStudioShare(row) ? 'live' : 'snapshot'; -}; - -// A public share: the Studio, read-only, on the stored copy. No account -// needed to look. A live share's data cards leave the server held and are -// filled in the browser on current data (sharedViewOf). +// A public share: the Studio, read-only, on the share dfo-apis holds. No +// account needed to look. A snapshot's data cards are drawn here from their +// component revisions; a live share's leave the server held and are filled +// in the browser on current data (src/lib/studio/slotCards). const SharePage = async ({ params }: SharePageProps) => { const { id } = await params; const row = await getShare(id); @@ -54,7 +44,7 @@ const SharePage = async ({ params }: SharePageProps) => { after(() => track( 'share.open', - { shareId: row.id, kind: row.kind, mode: modeOf(row), host }, + { shareId: row.id, kind: row.kind, mode: row.mode, host }, { headers: incoming }, ), ); @@ -65,7 +55,7 @@ const SharePage = async ({ params }: SharePageProps) => { redirect(`/data?share=${row.id}`); } - return ; + return ; }; export default SharePage; diff --git a/src/app/studio/[id]/page.tsx b/src/app/studio/[id]/page.tsx index 1d8a73b..e0f5c91 100644 --- a/src/app/studio/[id]/page.tsx +++ b/src/app/studio/[id]/page.tsx @@ -3,7 +3,6 @@ import type { Metadata } from 'next'; import Studio from '@/components/organisms/Studio/Studio'; import { getSession } from '@/lib/auth/session'; import { pickExamples } from '@/lib/studio/examples'; -import { liveSharingAvailable } from '@/lib/studio/liveShare'; export const metadata: Metadata = { title: 'DFO Studio', @@ -16,14 +15,7 @@ const StudioConversationPage = async ({ params }: { params: Promise<{ id: string const { id } = await params; const session = await getSession(); - return ( - - ); + return ; }; export default StudioConversationPage; diff --git a/src/app/studio/page.tsx b/src/app/studio/page.tsx index 23227ce..c81da0b 100644 --- a/src/app/studio/page.tsx +++ b/src/app/studio/page.tsx @@ -3,7 +3,6 @@ import type { Metadata } from 'next'; import Studio from '@/components/organisms/Studio/Studio'; import { getSession } from '@/lib/auth/session'; import { pickExamples } from '@/lib/studio/examples'; -import { liveSharingAvailable } from '@/lib/studio/liveShare'; export const metadata: Metadata = { title: 'DFO Studio', @@ -16,9 +15,7 @@ export const metadata: Metadata = { const StudioPage = async () => { const session = await getSession(); - return ( - - ); + return ; }; export default StudioPage; diff --git a/src/components/organisms/EmbedView/EmbedView.tsx b/src/components/organisms/EmbedView/EmbedView.tsx index 25e8e7b..e50380b 100644 --- a/src/components/organisms/EmbedView/EmbedView.tsx +++ b/src/components/organisms/EmbedView/EmbedView.tsx @@ -16,6 +16,7 @@ import { GRID_CELL } from '@/components/atoms/GridPattern/GridPattern'; import { useGridCell } from '@/hooks/useGridCell'; import type { SavedConversation } from '@/lib/chat/history'; import type { Widget } from '@/lib/chat/widgets'; +import { sharedCardAddress } from '@/lib/studio/cardDownload'; import { logRefusedDescriptions } from '@/lib/studio/infoPanel'; import { cardLiveOf, isLiveCanvas, type LiveCanvas, type SharedView } from '@/lib/studio/liveCards'; import { deriveTheme, DEFAULT_THEME, type ThemeDerived, themeVars } from '@/lib/studio/theme'; @@ -158,8 +159,15 @@ const EmbedView = ({ shared }: { shared: SharedView }) => { widget={infoWidget} open={infoCardId !== null} onOpenChange={(next) => !next && setInfoCardId(null)} - liveShareId={live?.shareId} - snapshot={shared.live ? undefined : shared} + shared={infoWidget && sharedCardAddress(shared, infoWidget.id)} + snapshot={ + shared.live + ? undefined + : { + id: shared.id, + servedAt: infoWidget ? shared.servedAt[infoWidget.id] : undefined, + } + } /> ); diff --git a/src/components/organisms/Studio/Studio.tsx b/src/components/organisms/Studio/Studio.tsx index 53f7a33..e22cfd4 100644 --- a/src/components/organisms/Studio/Studio.tsx +++ b/src/components/organisms/Studio/Studio.tsx @@ -37,6 +37,7 @@ import { latestTurnOf } from '@/lib/chat/canvas'; import type { SavedConversation } from '@/lib/chat/history'; import type { ShareKind } from '@/lib/chat/shareStore'; import { track } from '@/lib/events/client'; +import { sharedCardAddress } from '@/lib/studio/cardDownload'; import { stashGuestSession, takeGuestSession } from '@/lib/studio/guestStash'; import { logRefusedDescriptions } from '@/lib/studio/infoPanel'; import { GUEST_PROMPTS, promptEvent, type PromptIntent } from '@/lib/studio/promptAllowance'; @@ -61,12 +62,18 @@ export type StudioProps = { // conversation, read-only — no input, no card actions, no saving, no // way back and no account: logo, title, filters and the sheet. A `live` // share's data cards arrive held and are filled from the refresh route - // (see useLiveCards); `createdAt` is when the share was made. + // (see useLiveCards), and so are a snapshot's `undrawn` cards. `slots` + // names the slot that draws each card, whose rows a download serves. + // `servedAt` is when each snapshot card's slot began to show its data, + // and `createdAt` when the share was made. shared?: { id: string; kind: ShareKind; saved: SavedConversation; live?: boolean; + undrawn?: string[]; + slots?: Record; + servedAt?: Record; createdAt?: string; }; // /studio/: the conversation to open on arrival (a refresh, a link @@ -78,9 +85,6 @@ export type StudioProps = { // A gallery item: the same read-only sheet, plus the one action a // curated session exists for — carrying it on in your own Studio. continuable?: boolean; - // The site can refresh a live share's cards (`liveSharingAvailable`, - // read on the server), so the Share dialog offers Live Data. - liveSharing?: boolean; // The landing's example prompts for this visit (pickExamples, drawn by // the page). A share has no landing and passes none. examples?: readonly string[]; @@ -116,7 +120,6 @@ const Studio = ({ openId, guest = false, continuable = false, - liveSharing = false, examples = [], }: StudioProps) => { const sharedSaved = shared?.saved; @@ -478,8 +481,9 @@ const Studio = ({ const latestTurn = latestTurnOf(chat.widgets); // A live share's canvas: its data cards drawn on current data as each - // refresh answers, and their states. A snapshot's and the owner's own - // widgets pass through untouched. + // refresh answers, and their states. A snapshot's cards its page could + // not draw are filled the same way. The owner's own widgets pass + // through untouched. const { widgets: visibleWidgets, live: liveCanvas } = useLiveCards( shared, doc.visibleWidgets(chat.widgets), @@ -720,10 +724,17 @@ const Studio = ({ widget={infoWidget} open={infoCardId !== null} onOpenChange={(next) => !next && setInfoCardId(null)} - liveShareId={liveCanvas?.shareId} - // A share row's `live` is set either way (sharedViewOf). A - // gallery canvas leaves it unset and has no share link. - snapshot={shared?.live === false ? shared : undefined} + shared={infoWidget && sharedCardAddress(shared, infoWidget.id)} + // A share row's `live` is set either way (sharedViewOfRecord). + // A gallery canvas leaves it unset and has no share link. + snapshot={ + shared?.live === false + ? { + id: shared.id, + servedAt: infoWidget ? shared.servedAt?.[infoWidget.id] : undefined, + } + : undefined + } canShare={!readOnly} /> @@ -737,7 +748,7 @@ const Studio = ({ )} @@ -750,7 +761,6 @@ const Studio = ({ title={title} firstPrompt={firstPrompt} shareData={shareData} - liveSharing={liveSharing} createShare={save.createShare} guest={guest} /> diff --git a/src/components/organisms/Studio/canvas/cards/LivePlaceholder.tsx b/src/components/organisms/Studio/canvas/cards/LivePlaceholder.tsx index 4ecdbcf..e70ac0c 100644 --- a/src/components/organisms/Studio/canvas/cards/LivePlaceholder.tsx +++ b/src/components/organisms/Studio/canvas/cards/LivePlaceholder.tsx @@ -2,7 +2,6 @@ import classNames from 'classnames'; -import { useSecondsUntil } from '../../hooks/useSecondsUntil'; import cardStyles from './Card.module.scss'; import styles from './LivePlaceholder.module.scss'; @@ -20,16 +19,16 @@ type LivePlaceholderProps = { }; /** - * A live share's data card before its current data is drawn. While loading - * it shows the card's own outline with a sweep across it. On failure it - * shows a sentence in the card's place, and "Try again" when asking again - * can succeed (`liveCardFailure`). The button waits out the error the route - * has stored (`retryAt`) and says how long is left. Neither state shows a - * number. `data-live-status` and `data-live-code` name the state. + * A data card the browser fills, before its data is drawn: a live share's + * card, or a snapshot's card its page could not draw. While loading it + * shows the card's own outline with a sweep across it. On failure it shows + * a sentence in the card's place, and "Try again" when asking again can + * succeed (`liveCardFailure`). Neither state shows a number. + * `data-live-status` and `data-live-code` name the state. */ const LivePlaceholder = ({ title, kind, live }: LivePlaceholderProps) => { const { state } = live; - const wait = useSecondsUntil(state.status === 'error' ? state.retryAt : undefined); + const loading = live.snapshot ? 'Loading data' : 'Loading current data'; if (state.status === 'error') { const failure = liveCardFailure(state); @@ -46,13 +45,8 @@ const LivePlaceholder = ({ title, kind, live }: LivePlaceholderProps) => { {failure.message}

{failure.retry && ( - )} @@ -65,7 +59,7 @@ const LivePlaceholder = ({ title, kind, live }: LivePlaceholderProps) => { className={cardStyles.card} data-live-status="loading" aria-busy="true" - aria-label={title ? `${title}: loading current data` : 'Loading current data'} + aria-label={title ? `${title}: ${loading.toLowerCase()}` : loading} > {title ? (

{title}

@@ -80,7 +74,7 @@ const LivePlaceholder = ({ title, kind, live }: LivePlaceholderProps) => { aria-hidden="true" />

- Loading current data… + {loading}…

); diff --git a/src/components/organisms/Studio/hooks/useCardDownload.ts b/src/components/organisms/Studio/hooks/useCardDownload.ts index e80362e..1e70f5f 100644 --- a/src/components/organisms/Studio/hooks/useCardDownload.ts +++ b/src/components/organisms/Studio/hooks/useCardDownload.ts @@ -10,18 +10,18 @@ import { CardDownloadError, cardDownloadMessage, fetchCardData, - type LiveCardAddress, + type SharedCardAddress, } from '@/lib/studio/cardDownload'; /** * The download sheet's state for one card's own rows, over the export - * route, or over the share's data route when `live` names the card on a - * live share. Either route takes a format and the bundle flag only, so the + * route, or over the share's data route when `shared` names the card on a + * share. Either route takes a format and the bundle flag only, so the * sheet offers no extra columns, and keeps its choices for this card only. */ export const useCardDownload = ( ref: CardRef, - live: LiveCardAddress | null = null, + shared: SharedCardAddress | null = null, ): DownloadSheetState => { const [format, setFormat] = useState('csv'); const [bundle, setBundle] = useState(true); @@ -34,7 +34,7 @@ export const useCardDownload = ( setError(null); try { - const { blob, fileName } = await fetchCardData(ref, { format, bundle }, { live }); + const { blob, fileName } = await fetchCardData(ref, { format, bundle }, { shared }); downloadBlob(blob, fileName); } catch (caught) { diff --git a/src/components/organisms/Studio/hooks/useLiveCards.ts b/src/components/organisms/Studio/hooks/useLiveCards.ts index 5291813..5cef287 100644 --- a/src/components/organisms/Studio/hooks/useLiveCards.ts +++ b/src/components/organisms/Studio/hooks/useLiveCards.ts @@ -4,7 +4,7 @@ import { useEffect, useEffectEvent, useRef, useState } from 'react'; import type { Widget } from '@/lib/chat/widgets'; import { - isLiveDataCard, + cardsToFill, LIVE_CLIENT_CODES, type LiveCanvas, type LiveCardOutcome, @@ -16,23 +16,9 @@ import { const LOADING: LiveCardState = { status: 'loading' }; -/** The refresh client's clock, timer, and `fetch` in a browser. */ +/** The refresh client's `fetch` in a browser. */ export const browserLiveDeps: LiveDeps = { fetch: (...args) => fetch(...args), - now: () => Date.now(), - sleep: (ms, signal) => - new Promise((resolve) => { - const timer = window.setTimeout(resolve, ms); - - signal.addEventListener( - 'abort', - () => { - window.clearTimeout(timer); - resolve(); - }, - { once: true }, - ); - }), }; // A settled card holds its state and the widget drawn from a `ready` answer. @@ -53,25 +39,36 @@ const settledOf = (held: Widget | undefined, outcome: LiveCardOutcome): Settled }; /** - * The data cards of a live share, filled in the browser. `widgets` is the - * canvas as the share page sent it, whose data cards are held (see - * `sharedViewOf`). Each one is refreshed through - * `POST /api/share/[id]/cards/[cardId]/refresh` and drawn as soon as its own - * answer arrives. + * The data cards of a share filled in the browser: every data card of a + * live share, and the cards of a snapshot its page could not draw + * (`undrawn`). `widgets` is the canvas as the share page sent it, whose + * cards to fill are held (see `sharedViewOfRecord`). Each one is asked for + * through `GET /api/share/[id]/cards/[cardId]/refresh?slot=`, from the slot + * `share.slots` names for it, and drawn as soon as its own answer arrives. * - * Returns the widgets to render, with every refreshed card drawn in place - * of its held one, and the canvas's live state for the card components, - * whose `retry` puts one failed card back to loading and asks for it again. - * For a snapshot, or no share at all, it returns `widgets` itself and no - * live state, and makes no request. + * Returns the widgets to render, with every filled card drawn in place of + * its held one, and the canvas's state for the card components, whose + * `retry` puts one failed card back to loading and asks for it again. For a + * snapshot with nothing to fill, or no share at all, it returns `widgets` + * itself and no state, and makes no request. */ export const useLiveCards = ( - share: { id: string; live?: boolean; createdAt?: string } | undefined, + share: + | { + id: string; + live?: boolean; + undrawn?: string[]; + slots?: Record; + createdAt?: string; + } + | undefined, widgets: Widget[], ): { widgets: Widget[]; live: LiveCanvas | undefined } => { - const shareId = share?.live ? share.id : null; + const fill = cardsToFill(share, widgets); + const shareId = fill?.shareId ?? null; + const cardIds = fill?.cardIds ?? []; + const slots = share?.slots; const [settled, setSettled] = useState>({}); - const cardIds = shareId ? widgets.filter(isLiveDataCard).map((widget) => widget.id) : []; // The effect follows which cards are on the canvas, not the array's // identity, which changes on every render. const cardKey = JSON.stringify(cardIds); @@ -96,12 +93,13 @@ export const useLiveCards = ( void refreshLiveCards(browserLiveDeps, { shareId, cardIds: JSON.parse(cardKey) as string[], + slots, signal: controller.signal, onSettled: settle, }); return () => controller.abort(); - }, [shareId, cardKey]); + }, [shareId, cardKey, slots]); if (!shareId) { return { widgets, live: undefined }; @@ -122,6 +120,7 @@ export const useLiveCards = ( void refreshLiveCards(browserLiveDeps, { shareId, cardIds: [cardId], + slots, signal, onSettled: (id, outcome) => setSettled((prev) => ({ ...prev, [id]: settledOf(held, outcome) })), @@ -132,6 +131,7 @@ export const useLiveCards = ( widgets: widgets.map((widget) => settled[widget.id]?.widget ?? widget), live: { shareId, + snapshot: fill?.snapshot ?? false, createdAt: share?.createdAt, states: Object.fromEntries( cardIds.map((cardId) => [cardId, settled[cardId]?.state ?? LOADING]), diff --git a/src/components/organisms/Studio/hooks/useSecondsUntil.ts b/src/components/organisms/Studio/hooks/useSecondsUntil.ts deleted file mode 100644 index 233da79..0000000 --- a/src/components/organisms/Studio/hooks/useSecondsUntil.ts +++ /dev/null @@ -1,31 +0,0 @@ -'use client'; - -import { useEffect, useState } from 'react'; - -/** - * Whole seconds left until `at` (a `Date.now()` moment), rounded up, counting - * down once a second and 0 from then on. 0 when `at` is undefined. - */ -export const useSecondsUntil = (at: number | undefined): number => { - const [now, setNow] = useState(() => Date.now()); - - useEffect(() => { - if (at === undefined) { - return; - } - - const timer = window.setInterval(() => { - const moment = Date.now(); - - setNow(moment); - - if (moment >= at) { - window.clearInterval(timer); - } - }, 1000); - - return () => window.clearInterval(timer); - }, [at]); - - return at === undefined ? 0 : Math.max(0, Math.ceil((at - now) / 1000)); -}; diff --git a/src/components/organisms/Studio/modals/CardDownload.tsx b/src/components/organisms/Studio/modals/CardDownload.tsx index 03e992e..f88bcb8 100644 --- a/src/components/organisms/Studio/modals/CardDownload.tsx +++ b/src/components/organisms/Studio/modals/CardDownload.tsx @@ -5,7 +5,7 @@ import { useCardDownload } from '../hooks/useCardDownload'; import DatasetDownload from '@/components/organisms/DataSearch/DatasetDownload'; import type { CardInfo, CardRef, Widget } from '@/lib/chat/widgets'; -import { cardRefOf, type LiveCardAddress } from '@/lib/studio/cardDownload'; +import { cardRefOf, type SharedCardAddress } from '@/lib/studio/cardDownload'; import { formatWindow } from '@/lib/studio/period'; type DetailRow = { label: string; value: string }; @@ -46,13 +46,13 @@ const CardDetails = ({ info }: { info: CardInfo }) => ( const CardRows = ({ cardRef, info, - live, + shared, }: { cardRef: CardRef; info: CardInfo | null; - live: LiveCardAddress | null; + shared: SharedCardAddress | null; }) => { - const state = useCardDownload(cardRef, live); + const state = useCardDownload(cardRef, shared); return (
@@ -70,27 +70,23 @@ const CardRows = ({ * export route. `details` adds the Details rows. More Info leaves them out, * since its own rows above already describe the data. Renders nothing for a * card with no revision to export, such as a text card or one saved before - * cards kept their revision. `liveShareId` is the live share the card is - * shown on: the file then holds the rows the share last refreshed the card - * on, from the share's own data route. + * cards kept their revision. `shared` addresses the card on the share it is + * shown on, when a slot of the share draws it: the file then holds the rows + * the slot serves, from the share's own data route. */ const CardDownload = ({ widget, details = false, - liveShareId, + shared = null, }: { widget: Widget; details?: boolean; - liveShareId?: string; + shared?: SharedCardAddress | null; }) => { const cardRef = cardRefOf(widget); return cardRef ? ( - + ) : null; }; diff --git a/src/components/organisms/Studio/modals/CardInfoPanel.tsx b/src/components/organisms/Studio/modals/CardInfoPanel.tsx index 18d5aa9..c29d678 100644 --- a/src/components/organisms/Studio/modals/CardInfoPanel.tsx +++ b/src/components/organisms/Studio/modals/CardInfoPanel.tsx @@ -11,6 +11,7 @@ import SourceLink from '@/components/atoms/SourceLink/SourceLink'; import HoverPopup from '@/components/molecules/HoverPopup/HoverPopup'; import Modal from '@/components/molecules/Modal/Modal'; import type { CardInfo, Widget } from '@/lib/chat/widgets'; +import type { SharedCardAddress } from '@/lib/studio/cardDownload'; import { type DescribedView, type LegacyView, @@ -312,6 +313,10 @@ const LegacyPanel = ({ ); +/** The snapshot share a card is cited from: its id, and when the card's + * slot began to show the data it shows, an ISO timestamp. */ +export type CitedSnapshot = { id: string; servedAt?: string }; + // The link a snapshot share is cited by, read in the browser, where the // panel opens. const snapshotUrlOf = (shareId: string | undefined): string | null => @@ -319,10 +324,13 @@ const snapshotUrlOf = (shareId: string | undefined): string | null => ? `${window.location.origin}/s/${encodeURIComponent(shareId)}` : null; -// The day a snapshot share captured its data, in the reader's time zone, -// as its banner reads it. -const snapshotAsOfOf = (createdAt: string | undefined): string | null => { - const moment = createdAt ? new Date(createdAt) : null; +// The day a snapshot card's data dates from, in the reader's time zone: +// when its slot began to show it. Null when the share's history does not +// say, so the citation gives no date. The day the share was made is no +// stand-in: a live share made a snapshot later serves newer data. +const snapshotAsOfOf = (snapshot: CitedSnapshot | undefined): string | null => { + const at = snapshot?.servedAt; + const moment = at ? new Date(at) : null; return moment && !Number.isNaN(moment.getTime()) ? localDay(moment) : null; }; @@ -331,17 +339,18 @@ const snapshotAsOfOf = (createdAt: string | undefined): string | null => { * More Info, the back of the card. A card that carries a description shows * it as `DescribedPanel` lays it out, with no request. A card without one * shows the legacy panel. Download Data sits after the citation either way. - * On a live share (`liveShareId`) it serves the refreshed rows. The card's + * On a share whose slot draws the card (`shared`) it serves the rows that + * slot serves. The card's * owner (`canShare`) is told how to get a link to cite. The full citation * is retrieved today. On a snapshot share (`snapshot`) it names the share's - * link and the day the share captured its data. + * link and the day the card's data dates from. */ const CardInfoPanel = ({ info, widget, open, onOpenChange, - liveShareId, + shared, snapshot, canShare = false, }: { @@ -350,9 +359,9 @@ const CardInfoPanel = ({ widget: Widget | null; open: boolean; onOpenChange: (open: boolean) => void; - liveShareId?: string; - // The snapshot share the card is shown on, and when it was made. - snapshot?: { id: string; createdAt?: string }; + shared?: SharedCardAddress | null; + // The snapshot share the card is shown on. + snapshot?: CitedSnapshot; canShare?: boolean; }) => { const view = @@ -360,10 +369,10 @@ const CardInfoPanel = ({ panelViewOf(info, { retrievedOn: localDay(new Date()), snapshotUrl: snapshotUrlOf(snapshot?.id), - snapshotAsOf: snapshotAsOfOf(snapshot?.createdAt), + snapshotAsOf: snapshotAsOfOf(snapshot), canShare, }); - const download = widget && ; + const download = widget && ; return ( diff --git a/src/components/organisms/Studio/modals/ShareModal.tsx b/src/components/organisms/Studio/modals/ShareModal.tsx index 3dc8d1e..119b236 100644 --- a/src/components/organisms/Studio/modals/ShareModal.tsx +++ b/src/components/organisms/Studio/modals/ShareModal.tsx @@ -1,7 +1,6 @@ 'use client'; import classNames from 'classnames'; -import { useRouter } from 'next/navigation'; import { useEffect, useRef, useState } from 'react'; import { useCopied } from '../hooks/useCopied'; @@ -12,7 +11,7 @@ import Button from '@/components/atoms/Button/Button'; import Icon from '@/components/atoms/Icon/Icon'; import Modal from '@/components/molecules/Modal/Modal'; import SegmentedTabs from '@/components/molecules/SegmentedTabs/SegmentedTabs'; -import { GUEST_SHARES, type ShareMode, type StudioShareKind } from '@/lib/chat/shareMode'; +import type { ShareMode, StudioShareKind } from '@/lib/chat/shareMode'; import type { CardRef } from '@/lib/chat/widgets'; import { downloadBlob } from '@/lib/download'; import { DATASET_FORMATS, type DatasetFormat } from '@/lib/search/searchClient'; @@ -23,14 +22,12 @@ import { fetchCardData, } from '@/lib/studio/cardDownload'; import { - checkingText, openShareLink, type ShareLink, type ShareLinkDeps, shareLinkKey, shareModeFor, snapshotShare, - uncheckedText, withdrawalText, } from '@/lib/studio/shareDialog'; @@ -42,9 +39,10 @@ import { * when `offerLive` is set; Chat + Canvas is always a snapshot, and so is * every share when it is not. Links are made lazily, one per tab and mode * (`openShareLink` in src/lib/studio/shareDialog), and kept for the dialog's - * lifetime. A live link is shown once every data card it holds has been - * checked, and withdrawn when one cannot refresh. Data offers each card's - * stored rows as a file from the export route (src/lib/studio/cardDownload). + * lifetime. The site checks a live link's data cards before it makes the + * share, and a link whose cards cannot show live data is withdrawn. Data + * offers each card's stored rows as a file from the export route + * (src/lib/studio/cardDownload). */ export type ShareScope = { kind: 'whole' } | { kind: 'card'; id: string; title: string }; @@ -59,13 +57,13 @@ type ShareModalProps = { // The conversation's title (embed