MDBF-1237: Foundry in Buildbot

configuration/builders/definitions/foundry/builders.py

@@ -0,0 +1,105 @@
+from pathlib import Path
+
+import yaml
+
+from configuration.builders.base import GenericBuilder
+from configuration.builders.common import docker_config
+from configuration.builders.sequences.foundry import autobake, dispatcher
+
+with open(Path(__file__).parent / "foundry.yaml") as f:
+    _FOUNDRY_CONFIG = yaml.safe_load(f)
+
+_SEQUENCE_BY_PACKAGE_TYPE = {
+    "rpm": autobake.rpm,
+    "deb": autobake.deb,
+}
+_REPO_FILE_BY_PACKAGE_TYPE = {
+    "rpm": "MariaDB.repo",
+    "deb": "mariadb.sources",
+}
+# TEMPORARY: pull the MariaDB-devel/libmariadb-dev repo from production CI
+# rather than $ARTIFACTS_URL (dev), to build against real tarballs while this
+# pipeline is still dev-only. Revert to os.environ["ARTIFACTS_URL"] once done.
+_DEVEL_REPO_ARTIFACTS_URL = "https://ci.mariadb.org"
+# x86 worker images are a separate 32-bit tag (e.g. "debian12-386"), not part
+# of the amd64/aarch64 multi-arch manifest the other image tags resolve to,
+# and need an explicit --platform on pull to fetch that 32-bit variant.
+_ARCH_IMAGE_SUFFIX = {"x86": "-386"}
+_ARCH_PLATFORM = {"x86": "linux/386"}
+
+
+def foundry_scheduler_name(mariadb_version):
+    return f"foundry_{mariadb_version.replace('.', '_')}_scheduler"
+
+
+# Selectable via foundry_force_scheduler's "plugin" parameter.
+FOUNDRY_PLUGINS = _FOUNDRY_CONFIG["plugins"]
+
+
+FOUNDRY_BUILDERS_BY_ARCH = {}
+# Builders grouped by (ops, os version), regardless of arch.
+FOUNDRY_BUILDERS_BY_PACKAGE = {}
+for package_config in _FOUNDRY_CONFIG["packages"]:
+    ops = package_config["ops"]
+    version = package_config["version"]
+    package = f"{ops}-{version}"
+    package_type = package_config["type"]
+    for arch in package_config["arch"]:
+        image = f"{ops}{version}{_ARCH_IMAGE_SUFFIX.get(arch, '')}"
+        platform = _ARCH_PLATFORM.get(arch)
+        if package_type == "bintar":
+            # Bintar builds link against a MariaDB server bintar (see
+            # autobake.bintar) instead of installing -devel packages from an
+            # autobake builder's repo, so there's no repo_file_url to build.
+            sequence = autobake.bintar(
+                docker_config(image=image, platform=platform),
+                package_config["ci_bintar_builder"],
+            )
+        else:
+            sequence_fn = _SEQUENCE_BY_PACKAGE_TYPE[package_type]
+            repo_file = _REPO_FILE_BY_PACKAGE_TYPE[package_type]
+            # Server autobake builder that publishes
+            # MariaDB-devel/libmariadb-dev for this platform, e.g.
+            # "amd64-debian-12-deb-autobake" -- see BUILDERS_AUTOBAKE in
+            # constants.py.
+            autobake_builder = (
+                f"{arch}-{package_config['os_info_key']}-{package_type}-autobake"
+            )
+            repo_file_url = (
+                f"{_DEVEL_REPO_ARTIFACTS_URL}/%(prop:tarbuildnum)s/{autobake_builder}/{repo_file}"
+            )
+            sequence = sequence_fn(
+                docker_config(image=image, platform=platform), repo_file_url
+            )
+        builder = GenericBuilder(
+            name=f"foundry-{arch}-{ops}-{version}",
+            sequences=[sequence],
+        )
+        FOUNDRY_BUILDERS_BY_ARCH.setdefault(arch, []).append(builder)
+        FOUNDRY_BUILDERS_BY_PACKAGE.setdefault(package, []).append(builder)
+
+# Supported MariaDB versions: which packages get built for each, and the
+# tarbuildnum of the server CI build to install devel packages from.
+# Configured in configuration/builders/definitions/foundry/foundry.yaml.
+FOUNDRY_MARIADB_VERSIONS = _FOUNDRY_CONFIG["mariadb_versions"]
+for version_config in FOUNDRY_MARIADB_VERSIONS.values():
+    for package in version_config["packages"]:
+        assert package in FOUNDRY_BUILDERS_BY_PACKAGE, f"Unknown foundry package: {package}"
+
+DISPATCHER_BUILDER = GenericBuilder(
+    name="foundry-trigger-builders",
+    sequences=[
+        dispatcher.trigger_foundry(
+            [
+                (
+                    foundry_scheduler_name(version),
+                    {
+                        "mariadb_version": version,
+                        "tarbuildnum": version_config["tarbuildnum"],
+                    },
+                )
+                for version, version_config in FOUNDRY_MARIADB_VERSIONS.items()
+            ]
+        )
+    ],
+)

configuration/builders/definitions/foundry/foundry.yaml

@@ -0,0 +1,147 @@
+---
+# Plugins registered in https://github.com/vuvova/foundry (one top-level
+# directory each). Selectable via foundry_force_scheduler's "plugin" param
+# and passed straight to "cmake -P run.cmake <plugin>" in the child builders.
+# Note: the plugin's installed MTR "plugin/<name>/suite/" directory isn't
+# necessarily named the same (e.g. tidesql's CMake target is "tidesdb") --
+# RunPluginMTRSuite discovers that at runtime rather than assuming it matches.
+plugins:
+  - disabled-functions
+  - levenshtein
+  - tidesql
+
+# Foundry package builders: one builder per (package, arch) combination.
+# "version" + "ops" concatenate to the docker image tag (see os_info.yaml),
+# e.g. ops: ubuntu, version: "22.04" -> image "ubuntu22.04".
+# "os_info_key" is the matching top-level key in os_info.yaml -- it's used to
+# derive the real MariaDB server autobake builder name
+# ("{arch}-{os_info_key}-{type}-autobake", see constants.py) that publishes
+# the MariaDB.repo/mariadb.sources file we install -devel packages from. It
+# doesn't always match "{ops}-{version}" (e.g. centos-stream9, ubuntu-2204).
+packages:
+  - ops: centosstream
+    version: "9"
+    type: rpm
+    os_info_key: centos-stream9
+    arch: [amd64, aarch64]
+  - ops: centosstream
+    version: "10"
+    type: rpm
+    os_info_key: centos-stream10
+    arch: [amd64, aarch64]
+  - ops: rhel
+    version: "8"
+    type: rpm
+    os_info_key: rhel-8
+    arch: [amd64, aarch64]
+  - ops: rhel
+    version: "9"
+    type: rpm
+    os_info_key: rhel-9
+    arch: [amd64, aarch64]
+  - ops: rhel
+    version: "10"
+    type: rpm
+    os_info_key: rhel-10
+    arch: [amd64, aarch64]
+  - ops: debian
+    version: "11"
+    type: deb
+    os_info_key: debian-11
+    arch: [amd64, aarch64]
+  - ops: debian
+    version: "12"
+    type: deb
+    os_info_key: debian-12
+    arch: [amd64, aarch64, x86]
+  - ops: debian
+    version: "13"
+    type: deb
+    os_info_key: debian-13
+    arch: [amd64, aarch64]
+  - ops: ubuntu
+    version: "22.04"
+    type: deb
+    os_info_key: ubuntu-2204
+    arch: [amd64, aarch64]
+  - ops: ubuntu
+    version: "24.04"
+    type: deb
+    os_info_key: ubuntu-2404
+    arch: [amd64, aarch64]
+  - ops: ubuntu
+    version: "26.04"
+    type: deb
+    os_info_key: ubuntu-2604
+    arch: [amd64, aarch64]
+  # sles/opensuse images only exist for amd64 -- see os_info.yaml.
+  - ops: sles
+    version: "1507"
+    type: rpm
+    os_info_key: sles-1507
+    arch: [amd64]
+  - ops: sles
+    version: "1600"
+    type: rpm
+    os_info_key: sles-1600
+    arch: [amd64]
+  - ops: opensuse
+    version: "1600"
+    type: rpm
+    os_info_key: opensuse-1600
+    arch: [amd64]
+  # Bintar builds: the plugin is built against a MariaDB server bintar
+  # (fetched from ci_bintar_builder on ci.mariadb.org, same tarbuildnum)
+  # instead of installed -devel packages -- see autobake.bintar. Only one
+  # image builds bintars per MariaDB version range: centos7 up to 11.4,
+  # almalinux8 from 11.8 on (see mariadb_versions below).
+  - ops: centos
+    version: "7-bintar"
+    type: bintar
+    ci_bintar_builder: amd64-centos-7-bintar
+    arch: [amd64]
+  - ops: almalinux
+    version: "8-bintar"
+    type: bintar
+    ci_bintar_builder: amd64-almalinux-8-bintar
+    arch: [amd64]
+
+# Supported MariaDB versions: which packages get built for each, and the
+# tarbuildnum of the server CI build to install MariaDB-devel/libmariadb-dev
+# from (i.e. https://ci.mariadb.org/<tarbuildnum>/). Add or remove a version
+# here to change what "foundry-trigger-builders" dispatches.
+#
+# TODO: these tarbuildnum values are placeholders -- replace with the actual
+# last-known-good tarbuildnum for each branch.
+mariadb_versions:
+  "11.4":
+    tarbuildnum: 73276
+    packages:
+      - centosstream-9
+      - centosstream-10
+      - rhel-8
+      - rhel-9
+      - rhel-10
+      - debian-11
+      - debian-12
+      - ubuntu-22.04
+      - ubuntu-24.04
+      - centos-7-bintar
+  # "11.8":
+  #   tarbuildnum: 0
+  #   packages:
+  #     - centosstream-9
+  #     - centosstream-10
+  #     - rhel-8
+  #     - rhel-9
+  #     - rhel-10
+  #     - debian-11
+  #     - debian-12
+  #     - debian-13
+  #     - ubuntu-22.04
+  #     - ubuntu-24.04
+  #     - ubuntu-26.04
+  #     - sles-1507
+  #     - sles-1600
+  #     - opensuse-1600
+  #     - almalinux-8-bintar