Foundry deb/rpm triggers

Author: RazvanLiviuVarzaru

configuration/builders/definitions/foundry/builders.py

@@ -1,7 +1,54 @@
+from pathlib import Path
+
+import yaml
+
 from configuration.builders.base import GenericBuilder
-from configuration.builders.sequences.foundry import dispatcher
+from configuration.builders.sequences.foundry import autobake, dispatcher
+
+with open(Path(__file__).parent / "foundry.yaml") as f:
+    _FOUNDRY_CONFIG = yaml.safe_load(f)
+
+_SEQUENCE_BY_PACKAGE_TYPE = {
+    "rpm": autobake.rpm,
+    "deb": autobake.deb,
+}
+
+
+def foundry_scheduler_name(mariadb_version):
+    return f"foundry_{mariadb_version.replace('.', '_')}_scheduler"
+
+
+FOUNDRY_BUILDERS_BY_ARCH = {}
+# Builders grouped by (ops, os version), regardless of arch.
+FOUNDRY_BUILDERS_BY_PACKAGE = {}
+for package_config in _FOUNDRY_CONFIG["packages"]:
+    ops = package_config["ops"]
+    version = package_config["version"]
+    package = f"{ops}-{version}"
+    sequence_fn = _SEQUENCE_BY_PACKAGE_TYPE[package_config["type"]]
+    for arch in package_config["arch"]:
+        builder = GenericBuilder(
+            name=f"foundry-{arch}-{ops}-{version}",
+            sequences=[sequence_fn()],
+        )
+        FOUNDRY_BUILDERS_BY_ARCH.setdefault(arch, []).append(builder)
+        FOUNDRY_BUILDERS_BY_PACKAGE.setdefault(package, []).append(builder)
+
+# Supported MariaDB versions, and which packages get built for each one.
+# Configured in configuration/builders/definitions/foundry/foundry.yaml.
+FOUNDRY_MARIADB_VERSIONS = _FOUNDRY_CONFIG["mariadb_versions"]
+for packages in FOUNDRY_MARIADB_VERSIONS.values():
+    for package in packages:
+        assert package in FOUNDRY_BUILDERS_BY_PACKAGE, f"Unknown foundry package: {package}"
 
 DISPATCHER_BUILDER = GenericBuilder(
     name="foundry-trigger-builders",
-    sequences=[dispatcher.trigger_foundry()],
-)
+    sequences=[
+        dispatcher.trigger_foundry(
+            [
+                (foundry_scheduler_name(version), {"mariadb_version": version})
+                for version in FOUNDRY_MARIADB_VERSIONS
+            ]
+        )
+    ],
+)

configuration/builders/definitions/foundry/foundry.yaml

@@ -0,0 +1,26 @@
+---
+# Foundry package builders: one builder per (package, arch) combination.
+packages:
+  - ops: centosstream
+    version: "9"
+    type: rpm
+    arch:
+      - amd64
+      - aarch64
+
+  - ops: debian
+    version: "12"
+    type: deb
+    arch:
+      - amd64
+      - aarch64
+
+# Supported MariaDB versions, and which packages (by "{ops}-{version}") get
+# built for each one. Add or remove a version here to change what
+# "foundry-trigger-builders" dispatches.
+mariadb_versions:
+  "11.4":
+    - centosstream-9
+    - debian-12
+  "11.8":
+    - debian-12

configuration/builders/sequences/foundry/autobake.py

@@ -0,0 +1,25 @@
+from configuration.builders.infra.runtime import (
+    BuildSequence,
+    DockerConfig,
+    InContainer,
+)
+from configuration.steps.remote import PropFromShellStep, ShellStep
+from configuration.steps.commands.util import PrintEnvironmentDetails
+
+_MARIADB_VERSION_ENV = [("MARIADB_VERSION", "%(prop:mariadb_version)s")]
+
+
+def deb():
+    sequence = BuildSequence()
+    sequence.add_step(
+        ShellStep(command=PrintEnvironmentDetails(), env_vars=_MARIADB_VERSION_ENV)
+    )
+    return sequence
+
+
+def rpm():
+    sequence = BuildSequence()
+    sequence.add_step(
+        ShellStep(command=PrintEnvironmentDetails(), env_vars=_MARIADB_VERSION_ENV)
+    )
+    return sequence

configuration/builders/sequences/foundry/dispatcher.py

@@ -1,14 +1,11 @@
-from configuration.builders.infra.runtime import (
-    BuildSequence,
-    DockerConfig,
-    InContainer,
-)
-from configuration.steps.remote import PropFromShellStep, ShellStep
-from configuration.steps.commands.util import PrintEnvironmentDetails
+from configuration.builders.infra.runtime import BuildSequence
+from configuration.steps.commands import trigger
 
 
-
-def trigger_foundry():
+def trigger_foundry(trigger_specs):
+    # trigger_specs: list of (schedulername, properties) pairs -- see
+    # configuration/builders/definitions/foundry/builders.py for how the
+    # MariaDB-version/package matrix is turned into this list.
     sequence = BuildSequence()
-    sequence.add_step(ShellStep(command=PrintEnvironmentDetails()))
+    sequence.add_step(trigger.FoundryDispatch(trigger_specs))
     return sequence

configuration/schedulers/foundry.py

@@ -1,9 +1,11 @@
 import configuration.builders.definitions.foundry.builders as foundry_builders
 from buildbot.plugins import schedulers, util
 
-
-FOUNDRY_SCHEDULERS = []
-FOUNDRY_SCHEDULERS.append(
+# Force build entry point for the dispatcher. Must be loaded on whichever
+# master serves the www UI, since ForceScheduler availability is resolved
+# per-process (master.allSchedulers()), not shared via the DB.
+FOUNDRY_FORCE_SCHEDULERS = []
+FOUNDRY_FORCE_SCHEDULERS.append(
     schedulers.ForceScheduler(
         name="foundry_force_scheduler",
         builderNames=[foundry_builders.DISPATCHER_BUILDER.name],
@@ -17,4 +19,25 @@
             )
         ]
     )
-)
+)
+
+# One Triggerable per supported MariaDB version, holding whichever packages
+# that version builds -- see FOUNDRY_MARIADB_VERSIONS in
+# configuration/builders/definitions/foundry/foundry.yaml. The dispatcher
+# builder (foundry-trigger-builders) fires each of these once, setting
+# mariadb_version, via getSchedulersAndProperties in
+# configuration/steps/commands/trigger.py.
+# Must be loaded on whichever master executes the dispatcher build (i.e. the
+# master owning its workers), since Trigger steps look schedulers up on
+# their own process (master.scheduler_manager.namedServices).
+FOUNDRY_TRIGGERABLE_SCHEDULERS = [
+    schedulers.Triggerable(
+        name=foundry_builders.foundry_scheduler_name(version),
+        builderNames=[
+            builder.name
+            for package in packages
+            for builder in foundry_builders.FOUNDRY_BUILDERS_BY_PACKAGE[package]
+        ],
+    )
+    for version, packages in foundry_builders.FOUNDRY_MARIADB_VERSIONS.items()
+]

configuration/steps/commands/trigger.py

@@ -1,5 +1,6 @@
 from buildbot.plugins import steps
 from buildbot.process.properties import Property
+from buildbot.steps.trigger import Trigger as BuildbotTrigger
 from constants import SAVED_PACKAGE_BRANCHES
 from utils import (
     hasDockerLibrary,
@@ -90,6 +91,34 @@ def __init__(self):
         super().__init__(self.name, self.schedulername, self.doStepIf, properties)
 
 
+class _FoundryDispatchStep(BuildbotTrigger):
+    def __init__(self, trigger_specs, **kwargs):
+        self.trigger_specs = trigger_specs
+        super().__init__(**kwargs)
+
+    # Buildbot's dynamic-trigger extension point: lets one step fan out to a
+    # different set of schedulers/properties per (mariadb_version, package)
+    # pair, instead of needing one static Trigger step per version.
+    def getSchedulersAndProperties(self):
+        return self.trigger_specs
+
+
+class FoundryDispatch:
+    def __init__(self, trigger_specs):
+        # trigger_specs: list of (schedulername, properties) pairs, one per
+        # (mariadb_version, package) combination to fan out to.
+        self.trigger_specs = trigger_specs
+
+    def generate(self):
+        return _FoundryDispatchStep(
+            trigger_specs=self.trigger_specs,
+            name="Trigger Foundry Builders",
+            schedulerNames=sorted({name for name, _ in self.trigger_specs}),
+            waitForFinish=False,
+            updateSourceStamp=False,
+        )
+
+
 class Install(Server):
     def __init__(self):
         self.name = "Trigger Install Builders"

master-migration/master.cfg

@@ -30,6 +30,7 @@ from configuration.schedulers.connectors import (
     CONCPP_SCHEDULERS,
     CONODBC_SCHEDULERS,
 )
+from configuration.schedulers.foundry import FOUNDRY_TRIGGERABLE_SCHEDULERS
 from configuration.workers import worker
 from master_common import IS_CHECKCONFIG, base_master_config
 
@@ -48,7 +49,9 @@ c = BuildmasterConfig = base_master_config(config)
 # Do not import the schedulers from master_common since this master does not use any
 # TODO (Razvan): Schedulers/Services do not need to be defined for every master, this can be the job of a control-plane master
 c["schedulers"] = []
-c["schedulers"].extend(CONODBC_SCHEDULERS + CONCPP_SCHEDULERS + CONC_SCHEDULERS)
+c["schedulers"].extend(
+    CONODBC_SCHEDULERS + CONCPP_SCHEDULERS + CONC_SCHEDULERS + FOUNDRY_TRIGGERABLE_SCHEDULERS
+)
 
 ## ------------------------------------------------------------------- ##
 ##                            WORKER POOL                              ##
@@ -487,6 +490,17 @@ c["builders"].append(
     )
 )
 
+# Foundry package builders, triggered per MariaDB version by the dispatcher
+for arch, builders in foundry_builders.FOUNDRY_BUILDERS_BY_ARCH.items():
+    for builder in builders:
+        c["builders"].append(
+            builder.get_config(
+                workers=FOUNDRY_WORKERS_BY_ARCH.get(arch, []),
+                tags=["foundry"],
+                jobs=1,
+            )
+        )
+
 
 ## ------------------------------------------------------------------- ##
 ##                             REPORTERS                               ##

master-web/master.cfg

@@ -6,7 +6,7 @@ import os
 
 from buildbot.plugins import schedulers, util
 from flask import Flask, render_template
-from configuration.schedulers.foundry import FOUNDRY_SCHEDULERS
+from configuration.schedulers.foundry import FOUNDRY_FORCE_SCHEDULERS
 from master_common import base_master_config
 from pathlib import Path
 from utils import upstream_branch_fn
@@ -117,7 +117,7 @@ c["schedulers"] = []
 # looks at master.allSchedulers() in-memory, it is not DB-backed), so they
 # must be loaded on whichever master serves the www UI for the "Force" button
 # to show up, regardless of which master owns the builder's workers.
-c["schedulers"].extend(FOUNDRY_SCHEDULERS)
+c["schedulers"].extend(FOUNDRY_FORCE_SCHEDULERS)
 
 # upstream scheduling
 schedulerTarball = schedulers.AnyBranchScheduler(