feat: unify workload orchestration and profiles

Author: O2eg

.github/workflows/publish.yml

@@ -15,7 +15,7 @@ concurrency:
 jobs:
   build:
     name: Test and build distributions
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 45
 
     steps:
@@ -28,7 +28,7 @@ jobs:
           python-version: "3.10"
 
       - name: Install test and build dependencies
-        run: python -m pip install --upgrade pip ".[dev]"
+        run: python -m pip install --upgrade 'pip>=24.3,<26' ".[dev]"
 
       - name: Verify tag matches package version
         shell: bash
@@ -91,7 +91,7 @@ jobs:
   publish:
     name: Publish distributions to PyPI
     needs: build
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     timeout-minutes: 15
     environment:
       name: pypi

.github/workflows/tests.yml

@@ -2,14 +2,21 @@ name: tests
 
 on:
   push:
+    branches:
+      - "**"
   pull_request:
+  workflow_dispatch:
 
 permissions:
   contents: read
 
+concurrency:
+  group: tests-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   unit:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
         python-version: ["3.10", "3.12"]
@@ -24,14 +31,14 @@ jobs:
       - run: pytest -q
 
   package:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: unit
     steps:
       - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
       - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6
         with:
           python-version: "3.12"
-      - run: python -m pip install build twine
+      - run: python -m pip install 'build>=1.2,<2' 'twine>=6,<7'
       - run: python -m build
       - run: python -m twine check dist/*
       - name: Smoke-test installed wheel and bundled profiles
@@ -50,7 +57,7 @@ jobs:
             'import os; from pathlib import Path; root = Path(os.environ["RUNNER_TEMP"]) / "pg-workload-project/data"; assert len(list(root.glob("*/profile.yml"))) >= 7; assert not any(root.rglob("*.csv")); assert not any(root.rglob("*.tar.gz"))'
 
   postgres-18:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     needs: unit
     timeout-minutes: 30
     steps:

README.md

@@ -133,6 +133,8 @@ instead of `--passfile`; `PGPASSWORD` is accepted only as an admin-password comp
 
 `prepare-db --recreate` drops only the selected workload database after terminating its sessions;
 it does not drop the workload role.
+Unless `--extensions` is supplied, it creates both `pg_stat_statements` and
+`pg_buffercache`, matching the default pg_diag metric set used by pg_play.
 
 ## Targets
 
@@ -167,9 +169,9 @@ Use `--bin-dir /path/to/postgresql/bin` when client binaries are outside
 | `simple_stock_spec_symbols` | Identifier, Unicode, and parser edge cases | synthetic Python generator |
 | `imdb` | Movie-domain analytical joins and skew | synthetic Python generator |
 | `pagila` | Mixed Pagila OLTP | synthetic Python generator |
-| `many_objects` | Metadata-heavy schemas and partitions | SQL object creation only |
+| `many_objects` | Metadata-heavy schemas and partitions | scale-aware SQL object creation |
 | `emulate_errors` | Intentional SQL errors | SQL seed rows only |
-| `pss_overflow` | `pg_stat_statements` churn | SQL object creation; extension/preload required |
+| `pss_overflow` | `pg_stat_statements` churn | scaled Python generator; extension/preload required |
 
 No profile requires a dump, CSV file, or network download. The Pagila schema is redistributed
 under its upstream license; see `THIRD_PARTY_NOTICES.md`. Its rows are generated locally. The
@@ -284,8 +286,10 @@ For example, `--scale 2` approximately doubles scalable tables. Very small value
 profile-specific minimum so foreign-key structure and query selectivity remain meaningful in CI.
 Generators use deterministic seeds, preserve foreign-key relationships, and introduce skewed
 random distributions for hot entities, popularity, money, ratings, inventory, and customers.
-Profiles that model object count or failures (`many_objects`, `pss_overflow`, `emulate_errors`)
-have no Python generator and ignore `--scale` during preparation.
+`many_objects` scales its committed schema batches with `scale_repeat`; this
+also bounds DDL locks on small PostgreSQL stands. `pss_overflow` scales its
+generated table count (with a 50-table minimum). `emulate_errors` alone ignores
+`--scale` because it keeps a fixed, tiny seed.
 
 Installing a generated profile recreates its schema before filling it, so changing `--scale` and
 installing again produces a clean dataset rather than appending rows. Scheduler recovery reuses
@@ -315,6 +319,11 @@ pg-workload scheduler \
   --run-immediately
 ```
 
+`scheduler`, `start`, and their reviewed `plan --operation=scheduler` accept
+`--job-interval-seconds N` to apply one bounded interval to every selected
+profile. This is intended for short orchestrated observation windows; normal
+long-running installations should keep profile-specific intervals.
+
 Changes made by `enable`, `disable`, and `set-interval` are picked up without restarting the
 scheduler:
 

docs/pg_play-integration.md

@@ -12,9 +12,15 @@ pg-workload --machine --request-id workload-003 plan \
   --root workload --operation=install --profile pagila
 ```
 
+The capability document uses `pg_play/capabilities/v1`. Every command declares
+the common boolean fields `mutates_target`, `machine_output`, and
+`accepts_plan_hash`.
+Its `machine_interface` object records the canonical machine, request-id, and
+capability option names.
+
 `plan` hashes every selected profile file, selected jobs, scale, connection
 metadata without secrets, requested database changes, and scheduler state.
-Mutating `prepare-db`, `install`, `run`, `scheduler`, and `start` operations can
+Mutating `prepare-db`, `install`, `run`, and `start` operations can
 be guarded with the returned component plan hash.
 
 `start`, `status`, and `stop` provide a bounded background lifecycle useful to
@@ -25,6 +31,10 @@ For an orchestrated run, the hidden `--enable-selected` option is supplied to
 both `plan --operation=scheduler` and `start`. The plan hashes the prospective
 desired state; `start` verifies that hash, writes all selected enables under
 one state lock, and only then launches the scheduler.
+`--job-interval-seconds` is included in that same desired state, so a reviewed
+short diagnostic cadence cannot drift between plan and execution. Database
+preparation creates the default pg_diag extensions (`pg_stat_statements` and
+`pg_buffercache`) unless the caller explicitly supplies another extension set.
 
 Passwords are accepted only through the documented environment/passfile
 mechanisms. They are never accepted in the machine envelope or emitted to

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pg-workload"
-version = "0.2.0"
+version = "0.3.0"
 description = "PostgreSQL backend activity emulator for diverse plans, logs, and runtime statistics."
 readme = "README.md"
 requires-python = ">=3.10"

src/pg_workload/__init__.py

@@ -1,6 +1,6 @@
 """PostgreSQL workload generation and scheduling tools."""
 
-__version__ = "0.2.0"
+__version__ = "0.3.0"
 
 from pg_workload.assets import bundled_profiles_root, initialize_project
 from pg_workload.cli import build_parser

src/pg_workload/bundled/profiles/imdb/generator.py

@@ -71,28 +71,28 @@ def main() -> None:
 
         INSERT INTO cast_info (title_id, person_id, role_type, billing_order)
         SELECT
-            1 + ((g * 104729 - 1) % {titles}),
+            1 + ((g::bigint * 104729 - 1) % {titles}),
             1 + floor(power(random(), 1.8) * {people})::bigint,
             (ARRAY['actor', 'actress', 'director', 'writer', 'producer'])[1 + (g * 3) % 5],
             1 + (g % 20)
         FROM generate_series(1, {cast_rows}) AS g;
 
         INSERT INTO movie_keyword (title_id, keyword_id)
         SELECT
-            1 + ((g * 65537 - 1) % {titles}),
+            1 + ((g::bigint * 65537 - 1) % {titles}),
             1 + floor(power(random(), 2.3) * {keywords})::bigint
         FROM generate_series(1, {keyword_rows}) AS g;
 
         INSERT INTO movie_company (title_id, company_id, company_type)
         SELECT
-            1 + ((g * 32771 - 1) % {titles}),
+            1 + ((g::bigint * 32771 - 1) % {titles}),
             1 + floor(power(random(), 2.0) * {companies})::bigint,
             (ARRAY['production', 'distributor', 'effects', 'post-production'])[1 + (g % 4)]
         FROM generate_series(1, {company_rows}) AS g;
 
         INSERT INTO movie_info (title_id, info_type, info)
         SELECT
-            1 + ((g * 8191 - 1) % {titles}),
+            1 + ((g::bigint * 8191 - 1) % {titles}),
             (ARRAY['budget', 'gross', 'language', 'location'])[1 + (g % 4)],
             CASE g % 4
                 WHEN 0 THEN '$' || (100000 + (g::bigint * 7919) % 200000000)

src/pg_workload/bundled/profiles/many_objects/profile.yml

@@ -9,7 +9,8 @@ prepare:
       path: sql/prepare_schema_01.sql
     - type: sql
       path: sql/prepare_schema_02.sql
-      repeat: 5
+      repeat: 25
+      scale_repeat: true
 jobs:
   - name: metadata_scan
     type: pgbench

src/pg_workload/bundled/profiles/many_objects/sql/prepare_schema_02.sql

@@ -15,7 +15,9 @@ BEGIN
     FROM pg_namespace
     WHERE nspname LIKE 'many_objects_%';
 
-    FOR i IN (last_schema_index + 1)..(last_schema_index + 5) LOOP
+    -- One schema per invocation keeps DDL locks bounded.  The profile runner
+    -- scales the number of committed invocations with --scale.
+    FOR i IN (last_schema_index + 1)..(last_schema_index + 1) LOOP
         schema_name := 'many_objects_' || i;
 
         -- Create the schema
@@ -55,12 +57,35 @@ BEGIN
             END LOOP;
         END LOOP;
 
-        -- Create indexes on the partitioned table
-        EXECUTE format('
-            CREATE INDEX ON %I.%I (created_at);
-            CREATE INDEX ON %I.%I (data);
-        ', schema_name, table_name, schema_name, table_name);
+        -- PostgreSQL 11+ propagates parent indexes to every leaf partition.
+        -- PostgreSQL 10 rejects CREATE INDEX on a partitioned table; leaf
+        -- indexes are created below, after this DO block commits its DDL.
+        IF current_setting('server_version_num')::integer >= 110000 THEN
+            EXECUTE format('
+                CREATE INDEX ON %I.%I (created_at);
+                CREATE INDEX ON %I.%I (data);
+            ', schema_name, table_name, schema_name, table_name);
+        END IF;
 
         RAISE NOTICE 'Created schema % and partitioned table with partitions and subpartitions.', schema_name;
     END LOOP;
 END $$;
+
+-- psql \gexec runs every generated row as a separately committed command.
+-- This keeps the PostgreSQL 10 lock footprint bounded while preserving the
+-- indexed metadata shape used by the profile on newer releases.
+SELECT format(
+    'CREATE INDEX ON %I.%I (created_at); CREATE INDEX ON %I.%I (data);',
+    n.nspname,
+    c.relname,
+    n.nspname,
+    c.relname
+)
+FROM pg_catalog.pg_class c
+JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+WHERE current_setting('server_version_num')::integer < 110000
+  AND n.nspname LIKE 'many_objects_%'
+  AND c.relkind = 'r'
+  AND c.relname LIKE 'partitioned_table_partition_%_subpartition_%'
+ORDER BY n.nspname, c.relname
+\gexec