Skip to content

Commit 0cc0fad

Browse files
committed
feat: add portable web configurator
1 parent 79ad4ca commit 0cc0fad

47 files changed

Lines changed: 23458 additions & 190 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/publish.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
run: |
7272
ruff check .
7373
ruff format --check .
74-
pytest -m 'not integration'
74+
pytest -m 'not integration and not web'
7575
7676
- name: Build wheel and source distribution
7777
run: python -m build

‎.github/workflows/tests.yml‎

Lines changed: 106 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,71 @@ jobs:
2828
- run: python -m pip install -e '.[dev]'
2929
- run: ruff check .
3030
- run: ruff format --check .
31-
- run: pytest -m 'not integration'
31+
- run: pytest -m 'not integration and not web'
32+
33+
# The JavaScript port is compared with the Python reference on a pinned Node
34+
# version. It is a separate job on purpose: the ordinary Python matrix above
35+
# must stay usable without Node installed.
36+
web-parity:
37+
runs-on: ubuntu-24.04
38+
needs: unit
39+
steps:
40+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
41+
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
42+
with:
43+
python-version: "3.12"
44+
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
45+
with:
46+
node-version: "20"
47+
- run: python -m pip install -e '.[dev]'
48+
- name: Generated data is current
49+
run: python tools/export_web_data.py --check
50+
- name: JavaScript unit tests
51+
run: npm test
52+
- name: Differential parity against the Python reference
53+
env:
54+
PGC_REQUIRE_NODE: "1"
55+
run: pytest -m web
56+
# The differential suites prove the two implementations agree; this proves
57+
# the agreed answer starts a server. One major here, the full matrix stays
58+
# in the on-demand docker-integration job.
59+
- name: Boot a JavaScript-generated configuration
60+
env:
61+
PG_CONFIGURATOR_DOCKER_INTEGRATION: "1"
62+
PG_CONFIGURATOR_DOCKER_VERSIONS: "18"
63+
run: pytest -m integration -k "boots and javascript"
64+
65+
# The single-file page: it must build reproducibly and run offline in a real
66+
# browser. A page that only builds proves nothing.
67+
web-page:
68+
runs-on: ubuntu-24.04
69+
needs: unit
70+
strategy:
71+
matrix:
72+
engine: [chromium, firefox]
73+
steps:
74+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
75+
- uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
76+
with:
77+
python-version: "3.12"
78+
- run: python -m pip install --upgrade 'pip>=24.3,<26' playwright
79+
- run: python -m playwright install --with-deps ${{ matrix.engine }}
80+
- name: Build the page
81+
run: python3 web/build.py
82+
- name: Reproducible-build check
83+
run: |
84+
cp web/dist/pg-configurator.html /tmp/build1.html
85+
python3 web/build.py
86+
cmp /tmp/build1.html web/dist/pg-configurator.html
87+
- name: Browser regression (${{ matrix.engine }})
88+
env:
89+
PW_BROWSER: ${{ matrix.engine }}
90+
run: python3 web/tools/browser-smoke.py
91+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
92+
if: matrix.engine == 'chromium'
93+
with:
94+
name: pg-configurator-page
95+
path: web/dist/pg-configurator.html
3296

3397
distribution:
3498
runs-on: ubuntu-24.04
@@ -57,6 +121,47 @@ jobs:
57121
"${RUNNER_TEMP}/pg-configurator-wheel-smoke/bin/python" -c \
58122
'from pg_configurator import __version__; assert __version__'
59123
124+
# Live demo: the self-contained page, served as GitHub Pages.
125+
#
126+
# Requires Pages to be enabled with "GitHub Actions" as the source
127+
# (repo Settings -> Pages -> Build and deployment -> GitHub Actions).
128+
#
129+
# It lives here rather than in a workflow of its own so that `needs` is a real
130+
# gate. A separate Pages workflow ran its own subset of the checks and
131+
# deployed whatever it built, which meant a commit that failed lint or the
132+
# differential parity in this workflow still reached users. Nothing is built
133+
# here either: the artifact deployed is the exact file the browser regression
134+
# opened, downloaded from the job that opened it.
135+
pages:
136+
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
137+
needs: [unit, web-parity, web-page]
138+
runs-on: ubuntu-24.04
139+
permissions:
140+
contents: read
141+
pages: write
142+
id-token: write
143+
concurrency:
144+
group: pages
145+
cancel-in-progress: true
146+
environment:
147+
name: github-pages
148+
url: ${{ steps.deployment.outputs.page_url }}
149+
steps:
150+
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
151+
with:
152+
name: pg-configurator-page
153+
path: _page
154+
- name: Stage the demo site
155+
run: |
156+
mkdir -p _site
157+
cp _page/pg-configurator.html _site/index.html
158+
- uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5
159+
- uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4
160+
with:
161+
path: _site
162+
- id: deployment
163+
uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4
164+
60165
docker-integration:
61166
if: github.event_name == 'workflow_dispatch'
62167
runs-on: ubuntu-24.04

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ env/
1515
.eggs/
1616
build/
1717
dist/
18+
node_modules/
19+
web/pg-configurator.html
1820

1921
# Test and coverage artifacts
2022
.coverage

‎README.md‎

Lines changed: 107 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ CPU RAM disk duty PG major topology platform extension inventory
6060
+---------------+----------------+
6161
| | |
6262
v v v
63-
postgresql.conf Patroni JSON pg_configurator/v1 JSON
63+
postgresql.conf Patroni JSON pg_configurator/v2 JSON
6464
```
6565

6666
## Installation
@@ -257,7 +257,7 @@ machine running the command when their options are omitted.
257257
| `--db-cpu=CORES` | detected | Available CPU cores. Decimal cores and Kubernetes-style millicores such as `500m` are accepted. |
258258
| `--db-ram=SIZE` | detected | Physical RAM using IEC units such as `Mi`, `Gi`, or `Ti`. |
259259
| `--db-size=SIZE` | unknown | Optional logical database size. It selects a bounded `default_statistics_target` tier; omitting it uses only duty and resource capacity. |
260-
| `--db-disk-type=SATA\|SAS\|SSD\|NVME\|NETWORK` | `SAS` | Fallback storage class used only when no measured score is supplied. |
260+
| `--db-disk-type=SATA\|SAS\|SSD\|NVME\|NETWORK` | `SSD` | Fallback storage class used only when no measured score is supplied. |
261261
| `--disk-score=0..100` | inferred | Measured composite storage score; one score consistently drives planner, I/O, and vacuum cost. |
262262
| `--db-duty=financial\|oltp\|mixed\|statistic` | `mixed` | Workload composition used for connections, memory, autovacuum, timeouts, checkpoints, parallelism, logging, and network baselines. |
263263
| `--pg-version=9.6\|10..18` | `18` | Target PostgreSQL major and rule/snapshot compatibility contract. |
@@ -312,7 +312,7 @@ backend envelope exceeds 90% of available RAM.
312312

313313
## Artifact contract
314314

315-
JSON output uses `schema_version: pg_configurator/v1` and contains:
315+
JSON output uses `schema_version: pg_configurator/v2` and contains:
316316

317317
- normalized, typed inputs;
318318
- independent CPU, RAM, storage, connection, memory, worker, lock, WAL, and
@@ -329,9 +329,31 @@ JSON output uses `schema_version: pg_configurator/v1` and contains:
329329
- profile override history;
330330
- required extension metadata and whether availability was merely declared by
331331
the caller;
332-
- warnings and unverified assumptions;
332+
- `advisories`: what the tool has to say about the configuration it just
333+
generated, each one an object with a stable `code`, a `severity`, the
334+
`setting` and `actual` value it is about when there is one, and a `message`;
333335
- a flat `postgresql_conf` mapping for consumers that only need GUC values.
334336

337+
### Advisories
338+
339+
Every advisory is built from the finished configuration, after profiles and
340+
overrides have been applied, so `actual` is the value the emitted file carries
341+
and never a draft. They come in three severities, severest first:
342+
343+
| Severity | Means |
344+
| --- | --- |
345+
| `warning` | A real risk, or a conflict inside the result. |
346+
| `assumption` | A premise the calculation rests on and could not check. |
347+
| `info` | A boundary of what this tool does, or an explanation of a choice. |
348+
349+
`code` is stable across releases and is the field to route on; `message` is
350+
written for a person and is not. A configuration with no `warning` entries is
351+
the ordinary case — that separation is the point, because a tool that calls
352+
everything a warning teaches its reader to skip them.
353+
354+
Rendered `postgresql.conf` output carries the same text as comments, each
355+
prefixed with its severity.
356+
335357
Diagnostic messages are written to stderr. Stdout remains a valid JSON document
336358
when `--output-format=json` is selected.
337359

@@ -361,9 +383,15 @@ maintenance + autovacuum: 10%
361383
OS, backends and reserves: 45%
362384
```
363385

364-
The three configurable budgets may use at most 75% of available RAM. This
365-
preserves memory for PostgreSQL processes, extensions, WAL and lock structures,
366-
the operating system, and its filesystem cache. `effective_cache_size` includes
386+
Each budget has its own ceiling: `shared_buffers_part` may go up to 0.8,
387+
`client_mem_part` and `maintenance_mem_part` up to 0.4. `shared_buffers` is one
388+
allocation made at startup, while the other two are multiplied by the sessions
389+
using them at the same time, which is why it may claim the larger share.
390+
Together the three may use at most 85% of available RAM — below the 90% ceiling
391+
the calculated memory envelope is held to, because that envelope also counts
392+
lock tables and the per-backend reserve. This preserves memory for PostgreSQL
393+
processes, extensions, WAL and lock structures, the operating system, and its
394+
filesystem cache. `effective_cache_size` includes
367395
`shared_buffers` and only the OS-cache estimate left after concurrent query,
368396
maintenance, backend, lock, and logical-decoding reserves are subtracted.
369397

@@ -499,16 +527,20 @@ background headroom; parallel maintenance shares the parallel-worker pool.
499527
`--pitr-enabled` preserves the WAL level needed by a PITR design. It deliberately
500528
does not invent an `archive_command` or backup destination: credentials,
501529
retention, object storage, and archive transport belong to
502-
[pg_stand](https://github.com/O2eg/pg_stand). The artifact warns about this
503-
deployment boundary so WAL-level capability is
504-
not mistaken for a working backup chain.
530+
[pg_stand](https://github.com/O2eg/pg_stand). The artifact records this
531+
deployment boundary as an advisory so WAL-level capability is not mistaken for
532+
a working backup chain.
505533

506534
WAL retention uses the same bounded byte target on all major versions when
507535
replication is enabled:
508536

509537
```text
510538
desired retention = peak WAL bytes/second × tolerated replica outage
511-
retention = min(max(desired retention, 512 MiB), 40% of WAL disk budget)
539+
retained segments = max(1, min(
540+
ceil(max(desired retention, 512 MiB) / segment size),
541+
floor(40% of WAL disk budget / segment size) - 1
542+
))
543+
retention = retained segments × segment size
512544
513545
max_wal_size = min(
514546
max(peak WAL rate × checkpoint interval × 2, 1 GiB, 4 segments),
@@ -517,6 +549,13 @@ max_wal_size = min(
517549
min_wal_size = min(max(max_wal_size / 4, 2 segments), 4 GiB)
518550
```
519551

552+
Retention is decided in whole segments rather than in bytes because that is how
553+
PostgreSQL spends the disk: a byte request is converted down to whole segments,
554+
and the segment currently being written is retained on top of whatever was
555+
asked for. Both are subtracted here, so the 40% ceiling holds for what actually
556+
lands on `pg_wal`, not only for the number the file asks for. The eight-segment
557+
minimum on the budget is what keeps the retained count from reaching zero.
558+
520559
Configure it with `--peak-wal-rate`, `--replica-outage-tolerance`,
521560
`--wal-disk-budget`, and the actual `--wal-segment-size`. The disk budget must
522561
be at least 1 GiB and hold at least eight segments. Because the disk-budget cap
@@ -644,7 +683,7 @@ The baseline enables the logging collector with `csvlog`, daily or 256 MiB
644683
rotation, restrictive file permissions, bounded statement-duration thresholds,
645684
sampled shorter statements, lock waits, checkpoints, slow autovacuum, and temp
646685
files above 10 MiB. PostgreSQL cannot enforce a total log-directory size, so
647-
the artifact always warns that external retention for `pg_log` is required.
686+
the artifact always records that external retention for `pg_log` is required.
648687

649688
`pg_stat_statements` and `auto_explain` are always preloaded. Expensive
650689
`auto_explain` timing is disabled, only 1–5% of statements are sampled for
@@ -726,7 +765,7 @@ dependencies are an error. The CLI does not connect to PostgreSQL, inspect
726765
installed packages, load libraries, or read external-extension GUC metadata.
727766
Declared names are marked `declared_available`, not `verified`. Without the
728767
option, generation remains possible for offline planning, but every dependency
729-
is marked `unverified` and a warning is emitted.
768+
is marked `unverified` and the artifact records the assumption.
730769

731770
[pg_stand](https://github.com/O2eg/pg_stand) must build the inventory from the
732771
actual target and must not apply a candidate until package availability,
@@ -755,7 +794,10 @@ and the current
755794

756795
The profile intentionally requests compatibility settings including
757796
`ssl=off`, `row_security=off`, and `standard_conforming_strings=off`; each is
758-
reported as a warning. It does **not** adopt the performance-oriented
797+
reported as a warning. `row_security=off` does not read past a row-level
798+
security policy — a query that would have one applied fails with an error
799+
instead, unless the role owns the table or holds `BYPASSRLS`. The profile does
800+
**not** adopt the performance-oriented
759801
`synchronous_commit=off` recommendation because that can acknowledge recent
760802
transactions before they are durable. Patched-distribution-only parameters
761803
such as `enable_temp_memory_catalog` are not emitted into a vanilla PostgreSQL
@@ -770,7 +812,7 @@ Generation rejects:
770812

771813
- non-positive CPU or RAM;
772814
- reserves that leave no RAM for PostgreSQL;
773-
- invalid memory budgets or budgets using more than 75% of available RAM;
815+
- invalid memory budgets or budgets using more than 85% of available RAM;
774816
- `min` values greater than corresponding `max` values;
775817
- unknown PostgreSQL versions, profiles, or incompatible profile combinations;
776818
- unknown core GUCs and undeclared dotted extension GUCs;
@@ -823,8 +865,55 @@ config = configurator.make_conf("16", "64Gi", pg_version="18")
823865
artifact = configurator.build_artifact(config)
824866
```
825867

826-
`PGConfiguratorResult` is an instance-based result object; results and warnings
827-
are not shared between calls.
868+
`PGConfiguratorResult` is an instance-based result object; results and
869+
advisories are not shared between calls.
870+
871+
## JavaScript build
872+
873+
The same calculation runs in Node and in the browser. The Python package stays
874+
the reference: a differential suite drives both implementations with the same
875+
inputs and compares every generated setting, the advisories, the rendered
876+
`postgresql.conf`, the Patroni document, and the exit code and error text of a
877+
rejected command line.
878+
879+
```js
880+
import { createConfigurator, renderConf } from 'pg-configurator-web';
881+
882+
const pgc = await createConfigurator();
883+
const result = pgc.generate({ cpu_cores: 8, ram_value: '16Gi', pg_version: '18' });
884+
885+
result.config.shared_buffers; // '3622MB'
886+
result.advisories.filter((a) => a.severity === 'warning');
887+
renderConf(result, { version: '0.10.0', host: 'db-1' });
888+
```
889+
890+
`createConfigurator()` reads the bundled rule data from disk, which needs Node.
891+
A bundler or a browser imports the three JSON payloads itself and passes them
892+
instead — nothing else in the module touches the filesystem:
893+
894+
```js
895+
import { configuratorFromData } from 'pg-configurator-web';
896+
import rules from 'pg-configurator-web/data/rules.json' with { type: 'json' };
897+
import pgSettings from 'pg-configurator-web/data/pg_settings.json' with { type: 'json' };
898+
899+
const pgc = configuratorFromData({ rules: rules.payload, pgSettings: pgSettings.payload });
900+
```
901+
902+
`web/index.mjs` is the interface and is kept stable; the modules under
903+
`web/src/` mirror the Python source file by file so the two can be reviewed
904+
side by side, and are not part of it. The package also installs a
905+
`pg-configurator-js` command line that mirrors the Python CLI.
906+
907+
`--output-format=json` from the JavaScript build emits
908+
`schema_version: pg_configurator/preview-v1`: the same document as the
909+
`pg_configurator/v2` artifact, minus `artifact_hash`. The canonical hash is
910+
defined over the byte encoding of Python's `json.dumps`, down to how a float
911+
that happens to be integral is spelled, and this build does not reproduce those
912+
bytes. Emitting the same content under a name that promises a hash it does not
913+
carry is what a consumer cannot defend against, so it uses a name of its own.
914+
915+
The self-contained offline page is a separate artifact, built by
916+
`python3 web/build.py` and published from CI; it is not part of the npm package.
828917

829918
## Orchestrator integration
830919

@@ -860,7 +949,7 @@ python -m twine check dist/*
860949
```
861950

862951
Tagged releases are built and published through PyPI Trusted Publishing. A tag
863-
must match the package version, for example `v0.9.2`.
952+
must match the package version, for example `v0.10.0`.
864953

865954
## License and provenance
866955

0 commit comments

Comments
 (0)