MDBF-991 - Obtain failed test results and the associated output from MTR

configuration/builders/sequences/helpers.py

@@ -4,7 +4,7 @@
 from configuration.builders.infra.runtime import InContainer
 from configuration.steps.base import StepOptions
 from configuration.steps.commands.base import URL
-from configuration.steps.commands.mtr import MTRTest
+from configuration.steps.commands.mtr import MTRReporter, MTRTest
 from configuration.steps.commands.util import (
     CreateS3Bucket,
     DeleteS3Bucket,
@@ -48,6 +48,7 @@ def get_mtr_normal_steps(
                             MTROption(MTR.MAX_TEST_FAIL, 20),
                             MTROption(MTR.PARALLEL, jobs * 2),
                             MTROption(MTR.VARDIR, "/dev/shm/normal"),
+                            MTROption(MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "nm.xml"),
                         ],
                     ),
                 ),
@@ -86,6 +87,9 @@ def get_mtr_rocksdb_steps(
                             MTROption(MTR.VARDIR, "/dev/shm/rocksdb"),
                             MTROption(MTR.SUITE, "rocksdb*"),
                             MTROption(MTR.SKIP_TEST, "rocksdb_hotbackup*"),
+                            MTROption(
+                                MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "rocksdb.xml"
+                            ),
                         ],
                     ),
                 ),
@@ -123,6 +127,9 @@ def get_mtr_galera_steps(
                             MTROption(MTR.BIG_TEST, True),
                             MTROption(MTR.PARALLEL, jobs * 2),
                             MTROption(MTR.VARDIR, "/dev/shm/galera"),
+                            MTROption(
+                                MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "galera.xml"
+                            ),
                         ],
                         suite_collection=TestSuiteCollection(
                             [
@@ -174,6 +181,7 @@ def get_mtr_s3_steps(
                             MTROption(MTR.PARALLEL, jobs * 2),
                             MTROption(MTR.VARDIR, "/dev/shm/s3"),
                             MTROption(MTR.SUITE, "s3"),
+                            MTROption(MTR.XML_REPORT, MTR_PATH_TO_SAVE_LOGS / "s3.xml"),
                         ],
                     ),
                 ),
@@ -262,6 +270,14 @@ def add_test_suites_steps(
         )
     )
 
+    steps.append(
+        mtr_junit_reporter(
+            step_wrapping_fn=lambda step: InContainer(
+                docker_environment=config, step=step
+            ),
+        )
+    )
+
     return steps
 
 
@@ -286,3 +302,23 @@ def save_mtr_logs(
             ),
         ),
     )
+
+
+def mtr_junit_reporter(
+    step_wrapping_fn=lambda step: step,
+):
+    return step_wrapping_fn(
+        ShellStep(
+            command=MTRReporter(
+                workdir=PurePath("mtr/logs"),
+            ),
+            url=URL(
+                url=f"{os.environ['BUILDMASTER_URL']}/cr",
+                url_text="Test results",
+            ),
+            options=StepOptions(
+                alwaysRun=True, doStepIf=(lambda step: hasFailed(step))
+            ),
+            warn_on_fail=True,
+        ),
+    )

configuration/steps/commands/base.py

@@ -40,18 +40,22 @@ def as_cmd_arg(self) -> list[str]:
 
 class BashScriptCommand(Command):
     def __init__(
-        self, script_name: str, args: list[str] = None, user: str = "buildbot"
+        self,
+        script_name: str,
+        args: list[str] = None,
+        user: str = "buildbot",
+        workdir: PurePath = PurePath("."),
     ):
         name = f"Run {script_name}"
-        super().__init__(name=name, workdir=PurePath("."), user=user)
+        super().__init__(name=name, workdir=workdir, user=user)
         self.script_name = script_name
         self.args = args if args is not None else []
 
     def as_cmd_arg(self) -> list[str]:
         return [
             "bash",
             "-exc",
-            util.Interpolate(load_script(script_name=self.script_name)),
+            load_script(script_name=self.script_name),
             "--",
             *self.args,
         ]

configuration/steps/commands/mtr.py

@@ -1,7 +1,8 @@
+import os
 from pathlib import PurePath
 
 from buildbot.plugins import util
-from configuration.steps.commands.base import Command
+from configuration.steps.commands.base import BashScriptCommand, Command
 from configuration.steps.generators.mtr.generator import MTRGenerator
 
 
@@ -75,3 +76,25 @@ def _save_logs(self) -> str:
             find . -type f \( {patterns} \) -print0 | rsync -a --files-from=- --from0 ./ {self.save_logs_path}/
             exit 1
             """
+
+
+class MTRReporter(BashScriptCommand):
+    """
+    A command to transfer all the MTR JUnit test results to the mtr_junit_collector service.
+    Attributes:
+        directory (PurePath): The directory containing the MTR test results.
+    """
+
+    JUNIT_COLLECTOR_BASE_URL = os.environ.get("JUNIT_COLLECTOR_BASE_URL")
+
+    def __init__(self, workdir: PurePath = PurePath(".")):
+        base_url = self.JUNIT_COLLECTOR_BASE_URL
+        branch = util.Interpolate("%(prop:branch)s")
+        revision = util.Interpolate("%(prop:revision)s")
+        platform = util.Interpolate("%(prop:buildername)s")
+        bbnum = util.Interpolate("%(prop:buildnumber)s")
+        dir = "."
+
+        args = [base_url, branch, revision, platform, bbnum, dir]
+        super().__init__(script_name="mtr_reporter.sh", args=args, workdir=workdir)
+        self.name = "Save test results for CrossReference"

configuration/steps/commands/scripts/mtr_reporter.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Input variables
+BASE_URL="$1"     # e.g., 100.64.101.1:9990
+BRANCH="$2"
+REVISION="$3"
+PLATFORM="$4"
+BBNUM="$5"
+DIR="$6"          # Directory containing .xml files
+
+err() {
+  set +x
+  echo >&2 "ERROR: $*"
+  exit 1
+}
+
+bb_log_info() {
+  set +x
+  echo >&1 "INFO: $*"
+  set -x
+}
+
+UPLOAD_URL="${BASE_URL}/upload-test-results/"
+HEALTH_URL="${BASE_URL}/health"
+
+# Step 1: Health check before uploads
+command -v curl >/dev/null || err "curl not found"
+bb_log_info "Checking service health at $HEALTH_URL..."
+if ! curl "$HEALTH_URL" \
+  --max-time 5 \
+  --retry 3 \
+  --retry-max-time 0 \
+  --retry-delay 5 \
+  --retry-connrefused \
+  --fail-with-body; then
+    err "Service health check failed. Aborting uploads."
+fi
+bb_log_info "Service is healthy. Proceeding with uploads."
+
+# Step 2: Validate directory
+if [[ ! -d "$DIR" ]]; then
+  err "Error: directory '$DIR' does not exist"
+fi
+
+# Step 3: Find XML files
+shopt -s nullglob
+XML_FILES=("$DIR"/*.xml)
+shopt -u nullglob
+
+if (( ${#XML_FILES[@]} == 0 )); then
+  err "Error: no .xml files found in directory '$DIR'"
+fi
+
+# Step 4: Upload files and track failures
+ANY_FAILED=0
+
+for FILE in "${XML_FILES[@]}"; do
+  # Extract filename without extension for 'typ'
+  BASENAME="$(basename "$FILE" .xml)"
+  bb_log_info "Uploading $FILE (typ=$BASENAME)..."
+
+  if ! curl --max-time 120 --connect-timeout 10 --fail-with-body \
+    -X POST "$UPLOAD_URL" \
+    -F "branch=${BRANCH}" \
+    -F "revision=${REVISION}" \
+    -F "platform=${PLATFORM}" \
+    -F "bbnum=${BBNUM}" \
+    -F "typ=${BASENAME}" \
+    -F "file=@${FILE};type=application/xml"; then
+      bb_log_info "Upload failed for $FILE"
+      ANY_FAILED=1
+  else
+    bb_log_info "Upload succeeded for $FILE"
+  fi
+done
+
+# Step 5: Final result
+if ((ANY_FAILED != 0 )); then
+  err "One or more uploads failed."
+else
+  bb_log_info "All uploads succeeded."
+fi

configuration/steps/generators/mtr/options.py

@@ -34,6 +34,7 @@ class MTR(StrEnum):
     VIEW_PROTOCOL = "view-protocol"
     WITH_EMBEDDED = "embedded"
     VARDIR = "vardir"
+    XML_REPORT = "xml-report"
 
 
 # Extracted from ./mtr output manually before tests actually start.

configuration/steps/remote.py

@@ -1,5 +1,6 @@
 from buildbot.interfaces import IBuildStep
 from buildbot.plugins import steps, util
+from buildbot.process.results import SUCCESS, WARNINGS
 from configuration.steps.base import BaseStep, StepOptions
 from configuration.steps.commands.base import URL, Command, ShellCommandWithURL
 
@@ -15,8 +16,14 @@ class ShellStep(BaseStep):
         env_vars (list[tuple]): Environment variables to set for the command.
         url (str): Optional URL to associate with the step.
         urlText (str): Optional text for the URL. Defaults to the url itself.
+        timeout (int): Timeout for the command execution in seconds. Defaults to 1200 seconds.
+        warn_on_fail (bool): If True, treat non-zero return codes as warnings instead of failures.
+    Args:
     """
 
+    DEFAULT_DECODE_RC = {0: SUCCESS}
+    WARN_ON_FAIL_DECODE_RC = {0: SUCCESS, **{i: WARNINGS for i in range(1, 256)}}
+
     def __init__(
         self,
         command: Command,
@@ -25,6 +32,7 @@ def __init__(
         env_vars: list[tuple] = None,
         url: URL = None,
         timeout=1200,  # Default timeout in seconds
+        warn_on_fail=False,
     ):
         if env_vars is None:
             env_vars = []
@@ -36,6 +44,10 @@ def __init__(
         assert isinstance(command, Command)
         super().__init__(command.name, options)
         self.prefix_cmd = []
+        if warn_on_fail:
+            self.decode_return_code = self.WARN_ON_FAIL_DECODE_RC
+        else:
+            self.decode_return_code = self.DEFAULT_DECODE_RC
 
     def generate(self) -> IBuildStep:
         workdir = self._set_workdir()
@@ -48,6 +60,7 @@ def generate(self) -> IBuildStep:
             url=self.url,
             timeout=self.timeout,
             env={k: util.Interpolate(v) for k, v in self.env_vars},
+            decodeRC=self.decode_return_code,
         )
 
     def _set_workdir(self) -> str:

docker-compose/.env

@@ -2,6 +2,7 @@ TITLE='MariaDB CI'
 TITLE_URL='https://github.com/MariaDB/server'
 BUILDMASTER_URL='https://buildbot.mariadb.org/'
 BUILDMASTER_WG_IP='100.64.100.1'
+JUNIT_COLLECTOR_BASE_URL='http://100.64.100.1:9990'
 MQ_ROUTER_URL='ws://127.0.0.1:8080/ws'
 MASTER_PACKAGES_DIR='/mnt/autofs/master_packages'
 MASTER_CREDENTIALS_DIR='/srv/buildbot/master/master-credential-provider'

docker-compose/.env.dev

@@ -2,6 +2,7 @@ TITLE='MariaDB CI (DEV)'
 TITLE_URL='https://github.com/MariaDB/server'
 BUILDMASTER_URL='https://buildbot.dev.mariadb.org/'
 BUILDMASTER_WG_IP='100.64.101.1'
+JUNIT_COLLECTOR_BASE_URL='http://100.64.101.1:9990'
 MQ_ROUTER_URL='ws://127.0.0.1:8080/ws'
 MASTER_PACKAGES_DIR='/mnt/autofs/master_dev_packages'
 MASTER_CREDENTIALS_DIR='/srv/buildbot/master/master-credential-provider'