Skip to content

Commit 6cdd445

Browse files
RaviTejaKNTScodex
andcommitted
Fix stale cron markers and queue niche imports for the sync lock
Ignore pending markers without a live PID and clean up on termination. Use blocking advisory lock waits with bounded server timeouts and release the acquired lock on every exit path. Co-Authored-By: GPT-6.1-Sol <noreply@openai.com>
1 parent ba034fc commit 6cdd445

5 files changed

Lines changed: 158 additions & 48 deletions

File tree

‎content_docs/ad-networks/maintenance.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ Postgres owns existing public niche values. A full SQLite sync may seed a newly
3333

3434
The full sync now includes measured homepage title, description, h1, language, and source under `last_check_evidence.enrichment.homepage`. Homelab reads unsorted confirmed publishers and these facts directly from production. It never copies or pushes a second working store. `niche-sort.mjs` checks the exact input domain set, rejects duplicate/missing/extra domains, unsupported labels and fields, and invalid confidence before each atomic batch import. Low confidence suggestions remain private in homelab state; accepted results update only still-unsorted confirmed publishers. A changed OVH cohort cannot cause a stale full-row overwrite. Classified null results are retried after changed evidence or 180 days. Sites without usable homepage facts wait for a later homepage pass.
3535

36-
The model runs in a neutral temporary directory with tools disabled. Database credentials, tokens, passwords, secrets, and API keys are removed from its environment. Batch checkpoints live in `~/.local/state/stack127/ad-networks-niche/niche-results.json`, and the last run's result in `niche-status.json` beside it, outside Git. The import waits up to 60 minutes for the sync lock, polling every 5 seconds, because an OVH sync holds it for most of each cycle. Only the niche timer remains in `scripts/ad-networks/systemd/`. Keep the installed homelab backup timer until the original RDAP pass and gather supervisor finish: their already-loaded code still calls it at midnight. Do not stop the one-time RDAP pass.
36+
The model runs in a neutral temporary directory with tools disabled. Database credentials, tokens, passwords, secrets, and API keys are removed from its environment. Batch checkpoints live in `~/.local/state/stack127/ad-networks-niche/niche-results.json`, and the last run's result in `niche-status.json` beside it, outside Git. The import waits in the sync lock queue for up to 60 minutes, retrying after each 100-second lock timeout, because an OVH sync holds it for most of each cycle. Only the niche timer remains in `scripts/ad-networks/systemd/`. Keep the installed homelab backup timer until the original RDAP pass and gather supervisor finish: their already-loaded code still calls it at midnight. Do not stop the one-time RDAP pass.
3737

3838
Per-network Top niches now appears for any nonempty sorted sample. Shares use sorted sites, and the sorted count is visible. The 100-site/80% gates remain for broad niche comparisons, controls, and index/child loaders; the 500-site child publication gate remains too. Nothing in this round publishes those pages.
3939

‎scripts/ad-networks/niche-sort.mjs‎

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import { homedir, tmpdir } from 'node:os';
66
import { join, resolve } from 'node:path';
77
import { parseArgs } from 'node:util';
88
import { pathToFileURL } from 'node:url';
9-
import { setTimeout as delay } from 'node:timers/promises';
109
import { targetPool } from './runtime-db.mjs';
1110
import { publicNiche, validateNicheResult } from './niche-import.mjs';
1211
import { progressFile } from './enrichment-store.mjs';
@@ -61,27 +60,36 @@ export async function classifyNiches(input, rubric, options) {
6160
export async function importNiches(client, rows, dependencies = {}) {
6261
// Share the existing sync lock, then change only still-unsorted publishers.
6362
// A full OVH sync deliberately never updates existing niche values.
64-
// Short, nonblocking queries avoid the pool's 120-second query timeout
65-
// while OVH holds this lock for its full sync.
63+
// Queue behind other lock requests. Server-side timeouts keep each attempt
64+
// below the pool's 120-second query timeout while OVH finishes its full sync.
6665
const now = dependencies.now ?? (() => performance.now());
67-
const wait = dependencies.delay ?? delay;
6866
const deadline = now() + 60 * 60 * 1000;
69-
while (!(await client.query("select pg_try_advisory_lock(hashtext('ad-networks-reviewed-sync')) as locked")).rows[0].locked) {
70-
const remaining = deadline - now();
71-
if (remaining <= 0) throw new Error('Timed out waiting 60 minutes for ad-network sync lock');
72-
await wait(Math.min(5000, remaining));
73-
}
67+
let locked = false;
7468
try {
75-
await client.query('begin');
76-
const result = await client.query(`update public.ad_network_publishers p set niche=i.niche
77-
from jsonb_to_recordset($1::jsonb) i(domain text,niche text)
78-
where p.normalized_domain=i.domain and p.niche is null and i.niche is not null
79-
and exists(select 1 from public.ad_network_publisher_relationships r where r.publisher_id=p.id and r.status='confirmed')`,
80-
[JSON.stringify(rows.map(row => ({ domain: row.domain, niche: publicNiche(row) })))]);
81-
await client.query('commit');
82-
return result.rowCount;
83-
} catch (error) { await client.query('rollback').catch(() => {}); throw error; }
84-
finally { await client.query("select pg_advisory_unlock(hashtext('ad-networks-reviewed-sync'))").catch(() => {}); }
69+
try {
70+
while (!locked) {
71+
const remaining = deadline - now();
72+
if (remaining <= 0) throw new Error('Timed out waiting 60 minutes for ad-network sync lock');
73+
await client.query(`set lock_timeout = '${Math.max(1, Math.min(100000, Math.floor(remaining)))}ms'`);
74+
try {
75+
await client.query("select pg_advisory_lock(hashtext('ad-networks-reviewed-sync'))");
76+
locked = true;
77+
} catch (error) { if (error.code !== '55P03') throw error; }
78+
}
79+
} finally { await client.query('reset lock_timeout'); }
80+
try {
81+
await client.query('begin');
82+
const result = await client.query(`update public.ad_network_publishers p set niche=i.niche
83+
from jsonb_to_recordset($1::jsonb) i(domain text,niche text)
84+
where p.normalized_domain=i.domain and p.niche is null and i.niche is not null
85+
and exists(select 1 from public.ad_network_publisher_relationships r where r.publisher_id=p.id and r.status='confirmed')`,
86+
[JSON.stringify(rows.map(row => ({ domain: row.domain, niche: publicNiche(row) })))]);
87+
await client.query('commit');
88+
return result.rowCount;
89+
} catch (error) { await client.query('rollback').catch(() => {}); throw error; }
90+
} finally {
91+
if (locked) await client.query("select pg_advisory_unlock(hashtext('ad-networks-reviewed-sync'))").catch(() => {});
92+
}
8593
}
8694

8795
export async function sortNiches(options, dependencies = {}) {

‎scripts/ad-networks/ovh/run-job.sh‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,22 @@ set -u
66
job=$1; shift
77
cd /srv/stack127 || exit 1
88
set -a; . /srv/stack127-data/ad-networks.env; set +a
9-
pending=/tmp/adn-pending-$job
10-
if [ "$job" = verification ] && ls /tmp/adn-pending-* >/dev/null 2>&1; then exit 0; fi
9+
pending=/tmp/adn-pending-$job-$$
10+
if [ "$job" = verification ]; then
11+
for marker in /tmp/adn-pending-*; do
12+
[ -f "$marker" ] || continue
13+
IFS= read -r marker_pid < "$marker" || continue
14+
case "$marker_pid" in ''|*[!0-9]*|0*) continue ;; esac
15+
if kill -0 "$marker_pid" 2>/dev/null; then exit 0; fi
16+
done
17+
fi
1118
exec >>"/srv/stack127-data/logs/$job.log" 2>&1
12-
if [ "$job" != verification ]; then touch "$pending"; trap "rm -f $pending" EXIT; fi
19+
if [ "$job" != verification ]; then
20+
trap 'rm -f "$pending"' 0
21+
trap 'exit 130' INT
22+
trap 'exit 143' TERM
23+
printf '%s\n' "$$" > "$pending" || exit 1
24+
fi
1325
echo "== $(date -u +%FT%TZ) start $job $*"
1426
tries=0
1527
while :; do

‎tests/pages/ad-networks/niche-sort.test.mjs‎

Lines changed: 51 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,10 @@ test('niche imports withhold uncertain guesses, serialize against sync, and roll
2424
const calls = [];
2525
const client = { query: async (sql, args) => { calls.push({ sql, args }); return { rowCount: 1, rows: [{ locked: true }] }; } };
2626
assert.equal(await importNiches(client, [result('example.com'), result('example.org', 'gaming', 'low')]), 1);
27-
assert.deepEqual(JSON.parse(calls[2].args[0]), [{ domain: 'example.com', niche: 'food' }, { domain: 'example.org', niche: null }]);
28-
assert.match(calls[0].sql, /ad-networks-reviewed-sync/);
29-
assert.match(calls[2].sql, /p.niche is null/);
27+
const update = calls.find(call => call.sql.startsWith('update'));
28+
assert.deepEqual(JSON.parse(update.args[0]), [{ domain: 'example.com', niche: 'food' }, { domain: 'example.org', niche: null }]);
29+
assert.ok(calls.some(call => call.sql.includes("pg_advisory_lock(hashtext('ad-networks-reviewed-sync'))")));
30+
assert.match(update.sql, /p.niche is null/);
3031
calls.length = 0;
3132
await assert.rejects(importNiches({ query: async sql => { calls.push(sql); if (sql.startsWith('update')) throw Error('database failed'); return { rows: [{ locked: true }] }; } }, []));
3233
assert.ok(calls.includes('rollback')); assert.match(calls.at(-1), /unlock/);
@@ -39,43 +40,68 @@ test('small network niche shares use sorted sites while broad comparison gates s
3940
assert.deepEqual(adNetworkNicheShares([]), []);
4041
});
4142

42-
test('niche import waits through a twelve-minute sync without starting a transaction early', async () => {
43-
let elapsed = 0, attempts = 0;
44-
const calls = [], waits = [];
43+
test('niche import queues through a twelve-minute sync with bounded server lock timeouts', async () => {
44+
let elapsed = 0, attempts = 0, timeout;
45+
const calls = [];
4546
const client = { query: async sql => {
4647
calls.push(sql);
47-
if (sql.includes('pg_try_advisory_lock')) {
48+
if (sql.startsWith('set lock_timeout')) {
49+
timeout = Number(sql.match(/'(\d+)ms'/)[1]);
50+
assert.ok(timeout > 0 && timeout <= 100000);
51+
} else if (sql.includes('pg_advisory_lock')) {
4852
attempts++;
49-
return { rows: [{ locked: elapsed >= 12 * 60000 }] };
53+
const wait = Math.min(timeout, 12 * 60000 - elapsed);
54+
elapsed += wait;
55+
if (elapsed < 12 * 60000) throw Object.assign(Error('lock unavailable'), { code: '55P03' });
56+
} else {
57+
assert.ok(elapsed >= 12 * 60000);
5058
}
51-
assert.ok(elapsed >= 12 * 60000);
5259
return { rowCount: 1 };
5360
} };
54-
const imported = await importNiches(client, [result('example.com')], {
55-
now: () => elapsed, delay: async ms => { waits.push(ms); elapsed += ms; },
56-
});
57-
assert.equal(imported, 1);
58-
assert.equal(attempts, 145);
59-
assert.ok(waits.every(ms => ms === 5000));
61+
assert.equal(await importNiches(client, [result('example.com')], { now: () => elapsed }), 1);
62+
assert.equal(attempts, 8);
6063
assert.equal(calls.filter(sql => sql === 'begin').length, 1);
64+
assert.equal(calls[calls.indexOf('begin') - 1], 'reset lock_timeout');
6165
assert.match(calls.at(-1), /pg_advisory_unlock/);
6266
});
6367

64-
test('niche import stops after an hour of contention without importing or unlocking', async () => {
65-
let elapsed = 0;
68+
test('niche import stops after an hour of contention and resets the timeout without importing or unlocking', async () => {
69+
let elapsed = 0, timeout;
6670
const calls = [];
6771
await assert.rejects(importNiches({ query: async sql => {
6872
calls.push(sql);
69-
return { rows: [{ locked: false }] };
70-
} }, [], { now: () => elapsed, delay: async ms => { elapsed += ms; } }), /Timed out waiting 60 minutes/);
71-
assert.equal(elapsed, 60 * 60000);
72-
assert.ok(calls.every(sql => sql.includes('pg_try_advisory_lock')));
73+
if (sql.startsWith('set lock_timeout')) timeout = Number(sql.match(/'(\d+)ms'/)[1]);
74+
if (sql.includes('pg_advisory_lock')) {
75+
// Include time spent sending queries so the last wait has to be shorter.
76+
elapsed += timeout + 1;
77+
throw Object.assign(Error('lock unavailable'), { code: '55P03' });
78+
}
79+
} }, [], { now: () => elapsed }), /Timed out waiting 60 minutes/);
80+
assert.ok(timeout < 100000);
81+
assert.ok(elapsed >= 60 * 60000 && elapsed <= 60 * 60000 + 1);
82+
assert.equal(calls.at(-1), 'reset lock_timeout');
83+
assert.equal(calls.includes('begin'), false);
84+
assert.equal(calls.some(sql => sql.includes('pg_advisory_unlock')), false);
7385
});
7486

75-
test('a lock query failure fails immediately instead of retrying a broken connection', async () => {
76-
await assert.rejects(importNiches({ query: async () => { throw Error('connection lost'); } }, [], {
77-
delay: () => assert.fail('must not retry query errors'),
78-
}), /connection lost/);
87+
test('a lock query failure resets the timeout and fails immediately without retrying', async () => {
88+
const calls = [];
89+
await assert.rejects(importNiches({ query: async sql => {
90+
calls.push(sql);
91+
if (sql.includes('pg_advisory_lock')) throw Object.assign(Error('connection lost'), { code: '08006' });
92+
} }, []), /connection lost/);
93+
assert.equal(calls.filter(sql => sql.includes('pg_advisory_lock')).length, 1);
94+
assert.equal(calls.at(-1), 'reset lock_timeout');
95+
});
96+
97+
test('a timeout reset failure after acquisition still releases the advisory lock', async () => {
98+
const calls = [];
99+
await assert.rejects(importNiches({ query: async sql => {
100+
calls.push(sql);
101+
if (sql === 'reset lock_timeout') throw Error('reset failed');
102+
} }, []), /reset failed/);
103+
assert.equal(calls.includes('begin'), false);
104+
assert.match(calls.at(-1), /pg_advisory_unlock/);
79105
});
80106

81107
test('niche job keeps local completion and partial failure status and sends recovery alerts', async t => {
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import assert from 'node:assert/strict';
2+
import { spawn } from 'node:child_process';
3+
import { once } from 'node:events';
4+
import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises';
5+
import { join } from 'node:path';
6+
import { tmpdir } from 'node:os';
7+
import { setTimeout as delay } from 'node:timers/promises';
8+
import test from 'node:test';
9+
10+
async function fixture(t) {
11+
const directory = await mkdtemp(join(tmpdir(), 's127-ovh-wrapper-'));
12+
t.after(() => rm(directory, { recursive: true, force: true }));
13+
const repo = join(directory, 'repo'), data = join(directory, 'data'), bin = join(directory, 'bin');
14+
await mkdir(repo); await mkdir(join(data, 'logs'), { recursive: true }); await mkdir(bin);
15+
await writeFile(join(data, 'ad-networks.env'), `AD_NETWORK_DB_PATH=${join(data, 'unused.sqlite')}\nAD_NETWORKS_STATE_DIRECTORY=${data}\n`);
16+
const wrapper = join(directory, 'run-job.sh'), calls = join(directory, 'calls');
17+
const source = await readFile(new URL('../../../scripts/ad-networks/ovh/run-job.sh', import.meta.url), 'utf8');
18+
// Redirect host paths into this fixture and replace the collector launcher.
19+
// No database, production env file, or real collector is opened.
20+
await writeFile(wrapper, source.replaceAll('/srv/stack127-data', data).replaceAll('/srv/stack127', repo)
21+
.replaceAll('/tmp/adn-', join(directory, 'adn-')));
22+
await writeFile(join(bin, 'nice'), '#!/bin/sh\nprintf "run\\n" >> "$WRAPPER_CALLS"\nif [ "${WRAPPER_HOLD:-0}" = 1 ]; then while :; do sleep 1; done; fi\n', { mode: 0o755 });
23+
const launch = (job, hold = false) => {
24+
const child = spawn('/bin/sh', [wrapper, job], { detached: true,
25+
env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, WRAPPER_CALLS: calls, WRAPPER_HOLD: hold ? '1' : '0' },
26+
stdio: 'ignore' });
27+
t.after(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch { /* Already exited. */ } });
28+
return { child, completed: once(child, 'close') };
29+
};
30+
return { directory, calls, launch };
31+
}
32+
33+
test('verification ignores empty, invalid and dead PID markers but yields to a live wrapper', async t => {
34+
const { directory, calls, launch } = await fixture(t);
35+
const exited = spawn('/bin/sh', ['-c', 'exit 0']);
36+
await once(exited, 'close');
37+
for (const [name, value] of [['old', ''], ['invalid', 'invalid\n'], ['zero', '0000\n'], ['dead', `${exited.pid}\n`]]) {
38+
await writeFile(join(directory, `adn-pending-${name}`), value);
39+
}
40+
assert.deepEqual(await launch('verification').completed, [0, null]);
41+
assert.equal(await readFile(calls, 'utf8'), 'run\n');
42+
await writeFile(join(directory, 'adn-pending-live'), `${process.pid}\n`);
43+
assert.deepEqual(await launch('verification').completed, [0, null]);
44+
assert.equal(await readFile(calls, 'utf8'), 'run\n');
45+
});
46+
47+
for (const signal of ['SIGKILL', 'SIGTERM']) test(`verification recovers after a scheduled wrapper receives ${signal}`, async t => {
48+
const { directory, calls, launch } = await fixture(t);
49+
const scheduled = launch('traffic', true);
50+
for (let attempt = 0; ; attempt++) {
51+
try { if ((await readFile(calls, 'utf8')) === 'run\n') break; } catch (error) { if (error.code !== 'ENOENT') throw error; }
52+
assert.ok(attempt < 300, 'scheduled wrapper did not start');
53+
await delay(10);
54+
}
55+
const marker = join(directory, `adn-pending-traffic-${scheduled.child.pid}`);
56+
assert.equal(await readFile(marker, 'utf8'), `${scheduled.child.pid}\n`);
57+
assert.deepEqual(await launch('verification').completed, [0, null]);
58+
assert.equal(await readFile(calls, 'utf8'), 'run\n');
59+
process.kill(-scheduled.child.pid, signal);
60+
await scheduled.completed;
61+
assert.equal((await readdir(directory)).some(name => name.startsWith('adn-pending-')), signal === 'SIGKILL');
62+
assert.deepEqual(await launch('verification').completed, [0, null]);
63+
assert.equal(await readFile(calls, 'utf8'), 'run\nrun\n');
64+
});

0 commit comments

Comments
 (0)