test and save foundry packages

Author: RazvanLiviuVarzaru

configuration/builders/sequences/foundry/autobake.py

@@ -1,17 +1,25 @@
+import os
+
 from configuration.builders.infra.runtime import (
     BuildSequence,
     DockerConfig,
     InContainer,
 )
+from configuration.steps.commands.base import URL, BashCommand
 from configuration.steps.commands.download import GitInitFromCommit
-from configuration.steps.commands.foundry import BuildPlugin
+from configuration.steps.commands.foundry import (
+    BuildPlugin,
+    InstallBuiltPackages,
+    RunPluginMTRSuite,
+)
 from configuration.steps.commands.packages import (
     InstallDEBPackages,
     InstallRPMPackages,
+    SavePackages,
     SetupDEBRepoFromURL,
     SetupRPMRepoFromURL,
 )
-from configuration.steps.remote import ShellStep
+from configuration.steps.remote import PropFromShellStep, ShellStep
 
 _MARIADB_VERSION_ENV = [("MARIADB_VERSION", "%(prop:mariadb_version)s")]
 
@@ -30,9 +38,49 @@ def _clone_foundry_step(config: DockerConfig):
     )
 
 
+def _capture_foundry_revision_step(config: DockerConfig):
+    # foundry_force_scheduler never gives us a concrete revision (see
+    # _clone_foundry_step), so read back the commit that actually got
+    # checked out -- needed to tell apart saved packages built from
+    # different foundry/plugin source revisions.
+    return InContainer(
+        PropFromShellStep(
+            command=BashCommand(cmd="git rev-parse --short HEAD"),
+            property="foundry_revision",
+        ),
+        docker_environment=config,
+    )
+
+
+def _save_packages_step(config: DockerConfig, package_glob: str):
+    # The same builder gets triggered once per mariadb_version (and, on a
+    # different run, for a different plugin, or a different foundry commit)
+    # -- all three need to be in the destination path, or a later run
+    # silently overwrites an earlier one's saved packages.
+    # mariadb_version-tarbuildnum is kept as a single segment so it reads
+    # unambiguously as "the tarbuildnum for this mariadb_version", not some
+    # unrelated build number.
+    destination = (
+        "/packages/foundry/%(prop:mariadb_version)s-%(prop:tarbuildnum)s/%(prop:plugin)s"
+        "/%(prop:foundry_revision)s/%(prop:buildername)s"
+    )
+    url = (
+        f"{os.environ['ARTIFACTS_URL']}/foundry/%(prop:mariadb_version)s-%(prop:tarbuildnum)s"
+        "/%(prop:plugin)s/%(prop:foundry_revision)s/%(prop:buildername)s"
+    )
+    return InContainer(
+        ShellStep(
+            command=SavePackages(packages=[package_glob], destination=destination),
+            url=URL(url=url, url_text="Packages"),
+        ),
+        docker_environment=config,
+    )
+
+
 def deb(config: DockerConfig, repo_file_url: str):
     sequence = BuildSequence()
     sequence.add_step(_clone_foundry_step(config))
+    sequence.add_step(_capture_foundry_revision_step(config))
     sequence.add_step(
         InContainer(
             ShellStep(command=SetupDEBRepoFromURL(repo_file_url)),
@@ -53,12 +101,36 @@ def deb(config: DockerConfig, repo_file_url: str):
             docker_environment=config,
         )
     )
+    sequence.add_step(
+        InContainer(
+            ShellStep(command=InstallBuiltPackages("DEB")),
+            docker_environment=config,
+            container_commit=True,
+        )
+    )
+    sequence.add_step(_save_packages_step(config, "*.deb"))
+    sequence.add_step(
+        InContainer(
+            ShellStep(
+                command=InstallDEBPackages(packages=["mariadb-server", "mariadb-test"])
+            ),
+            docker_environment=config,
+            container_commit=True,
+        )
+    )
+    sequence.add_step(
+        InContainer(
+            ShellStep(command=RunPluginMTRSuite("/usr/share/mysql/mysql-test")),
+            docker_environment=config,
+        )
+    )
     return sequence
 
 
 def rpm(config: DockerConfig, repo_file_url: str):
     sequence = BuildSequence()
     sequence.add_step(_clone_foundry_step(config))
+    sequence.add_step(_capture_foundry_revision_step(config))
     sequence.add_step(
         InContainer(
             ShellStep(command=SetupRPMRepoFromURL(repo_file_url)),
@@ -79,4 +151,27 @@ def rpm(config: DockerConfig, repo_file_url: str):
             docker_environment=config,
         )
     )
+    sequence.add_step(
+        InContainer(
+            ShellStep(command=InstallBuiltPackages("RPM")),
+            docker_environment=config,
+            container_commit=True,
+        )
+    )
+    sequence.add_step(_save_packages_step(config, "*.rpm"))
+    sequence.add_step(
+        InContainer(
+            ShellStep(
+                command=InstallRPMPackages(packages=["MariaDB-server", "MariaDB-test"])
+            ),
+            docker_environment=config,
+            container_commit=True,
+        )
+    )
+    sequence.add_step(
+        InContainer(
+            ShellStep(command=RunPluginMTRSuite("/usr/share/mysql-test")),
+            docker_environment=config,
+        )
+    )
     return sequence

configuration/steps/commands/foundry.py

@@ -16,3 +16,63 @@ def as_cmd_arg(self) -> list[str]:
             "-exc",
             util.Interpolate(f"cmake -D{self.package_type}=1 -P run.cmake %(prop:plugin)s"),
         ]
+
+
+class InstallBuiltPackages(Command):
+    # Installs whatever run.cmake just built (copied next to run.cmake, see
+    # run.cmake's `file(COPY ${packages} DESTINATION "${b}/..")`).
+    def __init__(self, package_type: str, workdir: PurePath = PurePath(".")):
+        self.package_type = package_type
+        super().__init__(
+            name=f"Install built plugin packages ({package_type})",
+            workdir=workdir,
+            user="root",
+        )
+
+    def as_cmd_arg(self) -> list[str]:
+        if self.package_type == "RPM":
+            install_cmd = "dnf install -y ./*.rpm"
+        else:
+            install_cmd = "apt-get install -y ./*.deb"
+        return ["bash", "-exc", f"set -euo pipefail\n{install_cmd}"]
+
+
+class RunPluginMTRSuite(Command):
+    # Discovers the plugin's own MTR suite(s) the way mtr-suites.sh does --
+    # any directory under a mysql-test/ (or mysql-test/suite/) directory in
+    # the plugin's fetched source (<plugin>.build/) that has a suite.opt --
+    # copies them into the installed suite tree, and runs them. A plugin
+    # with no MTR suite is skipped rather than failing the build.
+    def __init__(self, mtr_base_dir: str, workdir: PurePath = PurePath(".")):
+        self.mtr_base_dir = mtr_base_dir
+        super().__init__(name="Run plugin MTR suite", workdir=workdir, user="root")
+
+    def as_cmd_arg(self) -> list[str]:
+        return [
+            "bash",
+            "-exc",
+            util.Interpolate(
+                f"""
+set -euo pipefail
+
+plugin_build="%(prop:plugin)s.build"
+suites=""
+for mt in $(find "$plugin_build" -type d -name mysql-test 2>/dev/null); do
+    for d in "$mt"/*/ "$mt"/suite/*/; do
+        [ -f "${{d}}suite.opt" ] || continue
+        name=$(basename "$d")
+        cp -r "$d" "{self.mtr_base_dir}/suite/$name"
+        suites="$suites,$name"
+    done
+done
+suites=$(echo "$suites" | sed 's/^,//')
+
+if [ -z "$suites" ]; then
+    echo "No MTR suite found for plugin %(prop:plugin)s -- skipping"
+    exit 0
+fi
+
+perl {self.mtr_base_dir}/mysql-test-run.pl --suite="$suites" --user=root
+"""
+            ),
+        ]