Define an MTR JUnit reporter (client)

Standalone build step that checks for xml's files in the mtr log path
and reports them back to the collector service.

The collector service is running on the buildmaster as a
docker container and its address is defined in the .env files

This is not a critical client service so just warn on step failure.
Common failures that can occur are: collector service is not available or the report is not a valid xml.

If all MTR steps are successful then naturally there's no failure to report
from the XML files so skip this step -> hasFailed(step)

Author: RazvanLiviuVarzaru

configuration/builders/sequences/helpers.py

@@ -4,7 +4,7 @@
 from configuration.builders.infra.runtime import InContainer
 from configuration.steps.base import StepOptions
 from configuration.steps.commands.base import URL
-from configuration.steps.commands.mtr import MTRTest
+from configuration.steps.commands.mtr import MTRTest, MTRReporter
 from configuration.steps.commands.util import (
     CreateS3Bucket,
     DeleteS3Bucket,
@@ -270,6 +270,14 @@ def add_test_suites_steps(
         )
     )
 
+    steps.append(
+        mtr_junit_reporter(
+            step_wrapping_fn=lambda step: InContainer(
+                docker_environment=config, step=step
+            ),
+        )
+    )
+
     return steps
 
 
@@ -294,3 +302,23 @@ def save_mtr_logs(
             ),
         ),
     )
+
+
+def mtr_junit_reporter(
+    step_wrapping_fn=lambda step: step,
+):
+    return step_wrapping_fn(
+        ShellStep(
+            command=MTRReporter(
+                directory=MTR_PATH_TO_SAVE_LOGS,
+            ),
+            url=URL(
+                url=f"{os.environ['BUILDMASTER_URL']}/cr",
+                url_text="Check MTR JUnit results on CrossReference",
+            ),
+            options=StepOptions(
+                alwaysRun=True, doStepIf=(lambda step: hasFailed(step))
+            ),
+            warn_on_fail=True,
+        ),
+    )

configuration/steps/commands/mtr.py

@@ -1,7 +1,9 @@
+import os
+
 from pathlib import PurePath
 
 from buildbot.plugins import util
-from configuration.steps.commands.base import Command
+from configuration.steps.commands.base import BashScriptCommand, Command
 from configuration.steps.generators.mtr.generator import MTRGenerator
 
 
@@ -75,3 +77,24 @@ def _save_logs(self) -> str:
             find . -type f \( {patterns} \) -print0 | rsync -a --files-from=- --from0 ./ {self.save_logs_path}/
             exit 1
             """
+
+
+class MTRReporter(BashScriptCommand):
+    """
+    A command to transfer all the MTR JUnit test results to the mtr_junit_collector service.
+    Attributes:
+        directory (PurePath): The directory containing the MTR test results.
+    """
+
+    JUNIT_COLLECTOR_BASE_URL = os.environ.get("JUNIT_COLLECTOR_BASE_URL")
+
+    def __init__(self, directory: PurePath):
+        base_url = self.JUNIT_COLLECTOR_BASE_URL
+        branch = util.Interpolate("%(prop:branch)s")
+        revision = util.Interpolate("%(prop:revision)s")
+        platform = util.Interpolate("%(prop:buildnumber)s")
+        bbnum = util.Interpolate("%(prop:buildnumber)s")
+        dir = str(directory)
+
+        args = [base_url, branch, revision, platform, bbnum, dir]
+        super().__init__(script_name="mtr_reporter.sh", args=args)

configuration/steps/commands/scripts/mtr_reporter.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Input variables
+BASE_URL="$1"     # e.g., 100.64.101.1:9990
+BRANCH="$2"
+REVISION="$3"
+PLATFORM="$4"
+BBNUM="$5"
+DIR="$6"          # Directory containing .xml files
+
+UPLOAD_URL="http://${BASE_URL}/upload-test-results/"
+HEALTH_URL="http://${BASE_URL}/health"
+
+# Step 1: Health check before uploads
+echo "Checking service health at $HEALTH_URL..."
+if ! curl "$HEALTH_URL" \
+  --max-time 5 \
+  --retry 3 \
+  --retry-max-time 0 \
+  --retry-delay 5 \
+  --retry-connrefused \
+  --fail-with-body > /dev/null; then
+    echo "Service health check failed. Aborting uploads."
+    exit 1
+fi
+echo "Service is healthy. Proceeding with uploads."
+
+# Step 2: Validate directory
+if [[ ! -d "$DIR" ]]; then
+  echo "Error: directory '$DIR' does not exist"
+  exit 1
+fi
+
+# Step 3: Find XML files
+shopt -s nullglob
+XML_FILES=("$DIR"/*.xml)
+shopt -u nullglob
+
+if [[ ${#XML_FILES[@]} -eq 0 ]]; then
+  echo "Error: no .xml files found in directory '$DIR'"
+  exit 1
+fi
+
+# Step 4: Upload files and track failures
+ANY_FAILED=0
+
+for FILE in "${XML_FILES[@]}"; do
+  # Extract filename without extension for 'typ'
+  BASENAME="$(basename "$FILE" .xml)"
+  echo "Uploading $FILE (typ=$BASENAME)..."
+
+  if ! curl --max-time 120 --connect-timeout 10 --fail-with-body \
+    -X POST "$UPLOAD_URL" \
+    -F "branch=${BRANCH}" \
+    -F "revision=${REVISION}" \
+    -F "platform=${PLATFORM}" \
+    -F "bbnum=${BBNUM}" \
+    -F "typ=${BASENAME}" \
+    -F "file=@${FILE};type=application/xml"; then
+      echo "Upload failed for $FILE"
+      ANY_FAILED=1
+  else
+    echo "Upload succeeded for $FILE"
+  fi
+done
+
+# Step 5: Final result
+if [[ $ANY_FAILED -ne 0 ]]; then
+  echo "One or more uploads failed."
+  exit 1
+else
+  echo "All uploads succeeded."
+  exit 0
+fi

docker-compose/.env

@@ -2,6 +2,7 @@ TITLE='MariaDB CI'
 TITLE_URL='https://github.com/MariaDB/server'
 BUILDMASTER_URL='https://buildbot.mariadb.org/'
 BUILDMASTER_WG_IP='100.64.100.1'
+JUNIT_COLLECTOR_BASE_URL='http://100.64.100.1:9990'
 MQ_ROUTER_URL='ws://127.0.0.1:8080/ws'
 MASTER_PACKAGES_DIR='/mnt/autofs/master_packages'
 MASTER_CREDENTIALS_DIR='/srv/buildbot/master/master-credential-provider'

docker-compose/.env.dev

@@ -2,6 +2,7 @@ TITLE='MariaDB CI (DEV)'
 TITLE_URL='https://github.com/MariaDB/server'
 BUILDMASTER_URL='https://buildbot.dev.mariadb.org/'
 BUILDMASTER_WG_IP='100.64.101.1'
+JUNIT_COLLECTOR_BASE_URL='http://100.64.101.1:9990'
 MQ_ROUTER_URL='ws://127.0.0.1:8080/ws'
 MASTER_PACKAGES_DIR='/mnt/autofs/master_dev_packages'
 MASTER_CREDENTIALS_DIR='/srv/buildbot/master/master-credential-provider'