Skip to content

Commit 28ce676

Browse files
RaviTejaKNTScodex
andcommitted
Make ad-network cache revision reads independent of dataset size
Track meaningful report changes with a transactional revision counter in the existing statement triggers. Keep other source branches and public projection semantics intact, fence concurrent writers during installation, and cover public changes and rollback with temporary SQL fixtures. Co-Authored-By: GPT-6.1-Sol <noreply@openai.com>
1 parent 6cdd445 commit 28ce676

2 files changed

Lines changed: 244 additions & 0 deletions

File tree

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
begin;
2+
set local lock_timeout='10s';
3+
set local statement_timeout='120s';
4+
5+
-- Fence in-flight writers before replacing their trigger function. There is no
6+
-- publisher backfill: the versioned baseline invalidates the old fingerprints.
7+
lock table public.ad_networks, public.ad_network_publishers,
8+
public.ad_network_publisher_relationships, public.ad_network_daily_counts,
9+
public.ad_network_relationship_events in share row exclusive mode;
10+
11+
create table public.stack127_ad_network_report_revision (
12+
id boolean primary key default true check (id),
13+
revision bigint not null default 0 check (revision >= 0)
14+
);
15+
insert into public.stack127_ad_network_report_revision(id) values(true);
16+
alter table public.stack127_ad_network_report_revision enable row level security;
17+
revoke all on public.stack127_ad_network_report_revision from public,anon,authenticated;
18+
grant select,update on public.stack127_ad_network_report_revision to service_role;
19+
20+
create or replace function public.stack127_cache_source_revisions(p_types text[] default null, p_slugs text[] default null)
21+
returns table(entity_type text, slug text, fingerprint text)
22+
language sql stable set search_path = pg_catalog, public as $$
23+
select 'page', page.slug,
24+
md5(to_jsonb(page)::text || coalesce(to_jsonb(tool)::text, '') ||
25+
coalesce((select to_jsonb(course)::text from public.stack127_course_pages course where course.page_id=page.id), '') ||
26+
coalesce((select to_jsonb(lesson)::text from public.stack127_course_lessons lesson where lesson.page_id=page.id), '') ||
27+
coalesce((select string_agg((to_jsonb(lesson)-'row_updated_at')::text || registry.is_published::text, '' order by lesson.display_order) from public.stack127_course_lessons lesson join public.stack127_pages registry on registry.id=lesson.page_id where lesson.course_page_id=page.id), '') ||
28+
coalesce((select to_jsonb(directory)::text from public.stack127_directory_pages directory where directory.page_id=page.id), '') ||
29+
coalesce((select to_jsonb(catalog)::text from public.stack127_catalog_pages catalog
30+
where catalog.page_id = page.id), '') ||
31+
coalesce((select string_agg(to_jsonb(member)::text, '' order by member.tool_page_id)
32+
from public.stack127_tool_catalog_memberships member where member.catalog_page_id = page.id), ''))
33+
from public.stack127_pages page
34+
left join public.stack127_tool_pages tool on tool.page_id = page.id
35+
where (p_types is null or 'page' = any(p_types)) and (p_slugs is null or page.slug = any(p_slugs))
36+
union all
37+
select 'self_host_app', app.slug, md5(to_jsonb(app)::text ||
38+
coalesce((select string_agg(to_jsonb(resource)::text, '' order by to_jsonb(resource)::text)
39+
from public.self_host_app_resource_estimates resource where resource.app_id = app.id), '') ||
40+
coalesce((select string_agg(to_jsonb(inst)::text, '' order by to_jsonb(inst)::text)
41+
from public.self_host_app_install_options inst where inst.app_id = app.id), '') ||
42+
coalesce((select string_agg(to_jsonb(link)::text, '' order by link.category_id)
43+
from public.self_host_app_category_links link where link.app_id = app.id), ''))
44+
from public.self_host_apps app where app.published
45+
and (p_types is null or 'self_host_app' = any(p_types)) and (p_slugs is null or app.slug = any(p_slugs))
46+
union all
47+
select 'cli_apps', 'cli-apps', md5(coalesce(string_agg(md5(to_jsonb(app)::text), '' order by app.slug), ''))
48+
from public.terminal_apps app
49+
having (p_types is null or 'cli_apps' = any(p_types)) and (p_slugs is null or 'cli-apps' = any(p_slugs))
50+
union all
51+
select 'self_host_catalog', 'all', md5(
52+
coalesce((select string_agg(md5(to_jsonb(g)::text), '' order by g.id) from public.github_projects g), '') ||
53+
coalesce((select string_agg(md5(to_jsonb(c)::text), '' order by c.id) from public.self_host_app_categories c), ''))
54+
where (p_types is null or 'self_host_catalog' = any(p_types)) and (p_slugs is null or 'all' = any(p_slugs))
55+
union all
56+
select 'platform_fees', section.id, md5(to_jsonb(section)::text ||
57+
coalesce((select string_agg(to_jsonb(entry)::text, '' order by entry.id)
58+
from public.platform_fee_entries entry where entry.section_id = section.id), ''))
59+
from public.platform_fee_sections section
60+
where (p_types is null or 'platform_fees' = any(p_types)) and (p_slugs is null or section.id = any(p_slugs))
61+
union all
62+
select 'ad_networks', network.slug,
63+
md5('ad-network-report-v2:' || revision.revision::text || ':' || network.slug)
64+
from public.ad_networks network
65+
cross join public.stack127_ad_network_report_revision revision
66+
where revision.id
67+
and (p_types is null or 'ad_networks'=any(p_types)) and (p_slugs is null or network.slug=any(p_slugs))
68+
union all
69+
-- All report routes share inputs, so every meaningful change invalidates the
70+
-- network fingerprints conservatively. Reads never scan publisher data.
71+
select 'ad_networks', 'all', md5('ad-network-report-v2:' || revision.revision::text || ':all')
72+
from public.stack127_ad_network_report_revision revision
73+
where revision.id
74+
and (p_types is null or 'ad_networks'=any(p_types)) and (p_slugs is null or 'all'=any(p_slugs))
75+
union all
76+
select 'launch_directories', 'launch-directories', md5(coalesce(string_agg(to_jsonb(directory)::text, '' order by directory.slug), ''))
77+
from public.launch_directories directory
78+
having (p_types is null or 'launch_directories' = any(p_types)) and (p_slugs is null or 'launch-directories' = any(p_slugs))
79+
$$;
80+
81+
-- Reuse the existing statement-level change detection, not row timestamps or
82+
-- row counts. The increment commits/rolls back with the data, and UPDATE locks
83+
-- serialize concurrent writers without losing increments. Work is limited to
84+
-- the statement's transition rows, regardless of the total dataset size.
85+
-- Keep stack127_directory_record_value unchanged. Daily snapshots omit only
86+
-- captured_at, exactly as the previous source fingerprint did.
87+
create or replace function public.stack127_touch_ad_network_reports()
88+
returns trigger language plpgsql security invoker set search_path=pg_catalog as $$
89+
declare changed boolean; route text; projection text;
90+
begin
91+
if tg_op='TRUNCATE' then
92+
changed := true;
93+
else
94+
projection := case when tg_table_name='ad_network_daily_counts' then 'to_jsonb(r)-''captured_at'''
95+
else format('public.stack127_directory_record_value(%L,to_jsonb(r))',tg_table_name) end;
96+
if tg_op='INSERT' then
97+
execute format('select exists(select 1 from new_rows r where (%s) is not null)',projection) into changed;
98+
elsif tg_op='DELETE' then
99+
execute format('select exists(select 1 from old_rows r where (%s) is not null)',projection) into changed;
100+
else
101+
execute format('with a as (select %s v from old_rows r), b as (select %s v from new_rows r)
102+
select exists(select 1 from ((select v from a except select v from b) union (select v from b except select v from a)) d where v is not null)',projection,projection) into changed;
103+
end if;
104+
end if;
105+
if not changed then return null; end if;
106+
update public.stack127_ad_network_report_revision set revision=revision+1 where id;
107+
if not found then raise exception 'Ad-network report revision baseline is missing'; end if;
108+
perform public.stack127_enqueue_revalidation('ad_networks','all',tg_table_name);
109+
for route in select slug from public.stack127_pages where slug='ad-networks' or slug like 'ad-networks/%' or slug='ad-network-checker' order by slug loop
110+
perform public.stack127_touch_page(route,tg_table_name);
111+
end loop;
112+
return null;
113+
end;
114+
$$;
115+
revoke all on function public.stack127_touch_ad_network_reports() from public,anon,authenticated;
116+
grant execute on function public.stack127_touch_ad_network_reports() to service_role;
117+
118+
do $$
119+
declare tab text;
120+
begin
121+
foreach tab in array array['ad_networks','ad_network_publishers','ad_network_publisher_relationships','ad_network_daily_counts','ad_network_relationship_events'] loop
122+
execute format('create trigger stack127_ad_reports_truncate after truncate on public.%I for each statement execute function public.stack127_touch_ad_network_reports()',tab);
123+
end loop;
124+
end;
125+
$$;
126+
127+
commit;
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import assert from 'node:assert/strict';
2+
import { readFile } from 'node:fs/promises';
3+
import test from 'node:test';
4+
import { targetPool } from '../../../scripts/ad-networks/runtime-db.mjs';
5+
6+
// All writes and new functions are session-local fixtures, rolled back. This
7+
// exercises the migration SQL without installing it or touching public data.
8+
test('report revision tracks public changes, ignores bookkeeping and rolls back with the data', {
9+
skip: process.env.AD_NETWORKS_TEST_ENVIRONMENT !== 'development',
10+
}, async () => {
11+
const migration = await readFile(new URL('../../../supabase/migrations/20261019102740_ad_network_report_revision_counter.sql', import.meta.url), 'utf8');
12+
const pool = targetPool({ environment: 'development' });
13+
const client = await pool.connect();
14+
const tables = ['ad_networks', 'ad_network_publishers', 'ad_network_publisher_relationships',
15+
'ad_network_daily_counts', 'ad_network_relationship_events'];
16+
try {
17+
await client.query('begin');
18+
await client.query(migration.match(/create table public.stack127_ad_network_report_revision \([\s\S]*?\n\);/)[0]
19+
.replace('create table public.', 'create temporary table '));
20+
await client.query('insert into pg_temp.stack127_ad_network_report_revision(id) values(true)');
21+
for (const table of tables) {
22+
// CTAS inherits column types, without defaults/sequences, public triggers
23+
// or constraints. All fixture IDs and values are supplied explicitly.
24+
await client.query(`create temporary table ${table} as select * from public.${table} with no data`);
25+
}
26+
await client.query('create temporary table stack127_pages(slug text)');
27+
await client.query('create temporary table report_queue(entity_type text,slug text,source text)');
28+
await client.query(`create function pg_temp.stack127_enqueue_revalidation(text,text,text) returns void language sql as $$
29+
insert into pg_temp.report_queue values($1,$2,$3) $$`);
30+
await client.query(`create function pg_temp.stack127_touch_page(text,text) returns void language plpgsql as $$begin return; end$$`);
31+
const trigger = migration.match(/create or replace function public.stack127_touch_ad_network_reports\(\)[\s\S]*?\$\$;/)[0]
32+
.replaceAll('public.stack127_touch_ad_network_reports', 'pg_temp.stack127_touch_ad_network_reports')
33+
.replaceAll('public.stack127_ad_network_report_revision', 'pg_temp.stack127_ad_network_report_revision')
34+
.replaceAll('public.stack127_enqueue_revalidation', 'pg_temp.stack127_enqueue_revalidation')
35+
.replaceAll('public.stack127_touch_page', 'pg_temp.stack127_touch_page')
36+
.replaceAll('public.stack127_pages', 'pg_temp.stack127_pages');
37+
await client.query(trigger);
38+
const revisions = migration.match(/create or replace function public.stack127_cache_source_revisions\([\s\S]*?\$\$;/)[0]
39+
.replace('public.stack127_cache_source_revisions', 'pg_temp.stack127_cache_source_revisions')
40+
.replaceAll('public.stack127_ad_network_report_revision', 'pg_temp.stack127_ad_network_report_revision')
41+
.replaceAll('public.ad_networks', 'pg_temp.ad_networks');
42+
await client.query(revisions);
43+
for (const table of tables) {
44+
await client.query(`create trigger report_insert after insert on pg_temp.${table}
45+
referencing new table as new_rows for each statement execute function pg_temp.stack127_touch_ad_network_reports()`);
46+
await client.query(`create trigger report_update after update on pg_temp.${table}
47+
referencing old table as old_rows new table as new_rows for each statement execute function pg_temp.stack127_touch_ad_network_reports()`);
48+
await client.query(`create trigger report_delete after delete on pg_temp.${table}
49+
referencing old table as old_rows for each statement execute function pg_temp.stack127_touch_ad_network_reports()`);
50+
await client.query(`create trigger report_truncate after truncate on pg_temp.${table}
51+
for each statement execute function pg_temp.stack127_touch_ad_network_reports()`);
52+
}
53+
const revision = async () => BigInt((await client.query('select revision from pg_temp.stack127_ad_network_report_revision')).rows[0].revision);
54+
const fingerprints = async slugs => (await client.query(`select slug,fingerprint
55+
from pg_temp.stack127_cache_source_revisions(array['ad_networks'],$1::text[]) order by slug`, [slugs ?? null])).rows;
56+
const change = async (sql, expected = true) => {
57+
const before = await revision(), fingerprintsBefore = await fingerprints();
58+
const queued = (await client.query('select count(*) as n from pg_temp.report_queue')).rows[0].n;
59+
await client.query(sql);
60+
assert.equal(await revision(), before + (expected ? 1n : 0n), sql);
61+
const after = await fingerprints();
62+
for (const previous of fingerprintsBefore) {
63+
const current = after.find(row => row.slug === previous.slug);
64+
if (current) assert.equal(current.fingerprint !== previous.fingerprint, expected, `${sql}: ${previous.slug}`);
65+
}
66+
assert.equal(BigInt((await client.query('select count(*) as n from pg_temp.report_queue')).rows[0].n),
67+
BigInt(queued) + (expected ? 1n : 0n));
68+
};
69+
assert.deepEqual((await fingerprints()).map(row => row.slug), ['all']);
70+
await change("insert into pg_temp.ad_networks(id,slug,name) values(1,'fixture-a','A'),(2,'fixture-b','B')");
71+
await change("insert into pg_temp.ad_network_publishers(id,normalized_domain,niche) values(1,'fixture.example','food')");
72+
await change(`insert into pg_temp.ad_network_publisher_relationships(ad_network_id,publisher_id,status,relationship_type,evidence)
73+
values(1,1,'confirmed','active_network_use','{}')`);
74+
await change(`insert into pg_temp.ad_network_daily_counts(ad_network_id,snapshot_on,listed_count,confirmed_count,stale_count)
75+
values(1,'2026-10-19',1,1,0)`);
76+
await change(`insert into pg_temp.ad_network_relationship_events(ad_network_id,publisher_id,event,observed_at,relationship_type)
77+
values(1,1,'joined','2026-10-19','active_network_use')`);
78+
assert.deepEqual((await fingerprints()).map(row => row.slug), ['all', 'fixture-a', 'fixture-b']);
79+
assert.equal((await fingerprints(['all'])).length, 1);
80+
assert.deepEqual(await fingerprints(['missing']), []);
81+
assert.equal((await client.query("select * from pg_temp.stack127_cache_source_revisions(array['missing'])")).rowCount, 0);
82+
83+
await change("update pg_temp.ad_network_publishers set next_check_at=now(),updated_at=now(),check_attempt_count=10,ads_txt_etag='new'", false);
84+
await change("update pg_temp.ad_network_publisher_relationships set consecutive_misses=2,evidence='{\"ignored\":true}'", false);
85+
await change('update pg_temp.ad_networks set updated_at=now()', false);
86+
await change('update pg_temp.ad_network_daily_counts set captured_at=now()', false);
87+
await change("update pg_temp.ad_network_publishers set niche='food'", false);
88+
await change("update pg_temp.ad_network_publishers set niche='tech' where id=-1", false);
89+
await change("update pg_temp.ad_network_publishers set niche='tech'");
90+
await change(`update pg_temp.ad_network_publishers set last_check_evidence='{"homepage":{"title":"Changed"}}'`);
91+
await change(`update pg_temp.ad_network_publishers set discovery_sources='["new-source"]'`);
92+
await change(`update pg_temp.ad_network_publisher_relationships set verification_method='direct_ads_txt'`);
93+
await change(`update pg_temp.ad_network_publisher_relationships set evidence='{"manager_domain":"manager.example"}'`);
94+
await change("update pg_temp.ad_networks set manager_domains=array['manager.example']");
95+
await change('update pg_temp.ad_network_daily_counts set listed_count=2');
96+
await change("update pg_temp.ad_network_relationship_events set observed_at='2026-10-20'");
97+
await change(`insert into pg_temp.ad_network_publisher_relationships(ad_network_id,publisher_id,status)
98+
values(2,1,'candidate')`, false);
99+
await change("update pg_temp.ad_network_publisher_relationships set status='needs_review' where ad_network_id=2", false);
100+
await change("update pg_temp.ad_network_publisher_relationships set status='confirmed' where ad_network_id=2");
101+
await change("update pg_temp.ad_network_publisher_relationships set status='needs_review' where ad_network_id=2");
102+
await change('delete from pg_temp.ad_network_publisher_relationships where ad_network_id=2', false);
103+
104+
const beforeRollback = await fingerprints(), versionBeforeRollback = await revision();
105+
await client.query('savepoint mutation');
106+
await change("update pg_temp.ad_network_publishers set niche='gaming'");
107+
await client.query('rollback to savepoint mutation');
108+
assert.equal(await revision(), versionBeforeRollback);
109+
assert.deepEqual(await fingerprints(), beforeRollback);
110+
for (const table of [...tables].reverse()) await change(`delete from pg_temp.${table}`);
111+
for (const table of tables) await change(`truncate pg_temp.${table}`);
112+
assert.deepEqual((await fingerprints()).map(row => row.slug), ['all']);
113+
} finally {
114+
await client.query('rollback').catch(() => {});
115+
client.release(); await pool.end();
116+
}
117+
});

0 commit comments

Comments
 (0)