Build a JUnit XML report from the results of a test run

cmd/pipelines/test_junit.go

@@ -0,0 +1,160 @@
+package pipelines
+
+import (
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"github.com/databricks/cli/cmd/root"
+	"github.com/databricks/cli/libs/atomicfile"
+)
+
+type junitTestSuites struct {
+	XMLName xml.Name         `xml:"testsuites"`
+	Suites  []junitTestSuite `xml:"testsuite"`
+}
+
+type junitTestSuite struct {
+	Name     string          `xml:"name,attr"`
+	Tests    int             `xml:"tests,attr"`
+	Failures int             `xml:"failures,attr"`
+	Errors   int             `xml:"errors,attr"`
+	Skipped  int             `xml:"skipped,attr"`
+	Cases    []junitTestCase `xml:"testcase"`
+}
+
+type junitTestCase struct {
+	ClassName string       `xml:"classname,attr,omitempty"`
+	Name      string       `xml:"name,attr"`
+	Time      string       `xml:"time,attr,omitempty"`
+	Failure   *junitResult `xml:"failure"`
+	Error     *junitResult `xml:"error"`
+	Skipped   *junitResult `xml:"skipped"`
+}
+
+type junitResult struct {
+	Message string `xml:"message,attr,omitempty"`
+	Text    string `xml:",chardata"`
+}
+
+// junitCaseName splits a pytest test ID the way pytest's JUnit report does: the classname
+// is the module path plus any class names, joined by dots. See
+// https://docs.pytest.org/en/stable/how-to/output.html#creating-junitxml-format-files.
+// A parameter ID in brackets, such as test_cast[a::int], stays with the test name. Brackets are
+// looked for only after the file path, which can contain them too.
+func junitCaseName(testID string) (string, string) {
+	file, node, hasNode := strings.Cut(testID, "::")
+	module := strings.ReplaceAll(strings.TrimSuffix(strings.TrimPrefix(file, "/"), ".py"), "/", ".")
+	if !hasNode {
+		return "", module
+	}
+	params := ""
+	if i := strings.Index(node, "["); i >= 0 {
+		node, params = node[:i], node[i:]
+	}
+	parts := strings.Split(node, "::")
+	parts[len(parts)-1] += params
+	classes := append([]string{module}, parts[:len(parts)-1]...)
+	return strings.Join(classes, "."), parts[len(parts)-1]
+}
+
+func newJUnitResult(text string) *junitResult {
+	message, _, _ := strings.Cut(text, "\n")
+	return &junitResult{Message: message, Text: text}
+}
+
+// junitNoResult is the error message of a collected case that has no final outcome.
+const junitNoResult = "no result received"
+
+// newJUnitReport has one entry per collected case; a case without a final outcome is an error,
+// because CI tools treat skipped entries as passing.
+// A non-empty problem, such as an incomplete run, adds a failing entry for the update so the
+// report never passes by omission.
+// Cases are named relative to projectFolder, as pytest names them, so that the names do not depend on
+// the working directory.
+func newJUnitReport(suiteName, updateID, projectFolder string, run *testRun, problem string) junitTestSuites {
+	suite := junitTestSuite{Name: suiteName}
+	for _, c := range run.cases {
+		entry := junitTestCase{}
+		entry.ClassName, entry.Name = junitCaseName(relativeTestID(projectFolder, c.id))
+		if d, ok := c.duration(); ok {
+			entry.Time = fmt.Sprintf("%.3f", d.Seconds())
+		}
+
+		details := ""
+		if c.errorDetail != nil {
+			details = errorDetailText(c.errorDetail)
+		}
+		if c.detailsTruncated {
+			details = strings.TrimPrefix(details+"\n(details truncated)", "\n")
+		}
+		switch c.status {
+		case testCasePassed:
+		case testCaseFailed:
+			entry.Failure = newJUnitResult(details)
+			suite.Failures++
+		case testCaseError:
+			entry.Error = newJUnitResult(details)
+			suite.Errors++
+		case testCaseSkipped:
+			entry.Skipped = &junitResult{}
+			suite.Skipped++
+		default:
+			entry.Error = &junitResult{Message: junitNoResult}
+			suite.Errors++
+		}
+		suite.Cases = append(suite.Cases, entry)
+	}
+
+	if problem != "" {
+		suite.Cases = append(suite.Cases, junitTestCase{
+			Name:  "update " + updateID,
+			Error: newJUnitResult(problem),
+		})
+		suite.Errors++
+	}
+	suite.Tests = len(suite.Cases)
+	return junitTestSuites{Suites: []junitTestSuite{suite}}
+}
+
+// testReport writes the JUnit report of a test run to path, if --junit-xml is set.
+type testReport struct {
+	path          string
+	out           io.Writer
+	suiteName     string
+	projectFolder string
+}
+
+// finish writes the report of a run that ended with runErr. A write error is joined with runErr,
+// so that neither is lost.
+func (r *testReport) finish(updateID string, run *testRun, problem string, runErr error) error {
+	if r.path == "" {
+		return runErr
+	}
+	if err := writeJUnitReport(r.path, newJUnitReport(r.suiteName, updateID, r.projectFolder, run, problem)); err != nil {
+		if runErr == nil || errors.Is(runErr, root.ErrAlreadyPrinted) {
+			return err
+		}
+		return errors.Join(runErr, err)
+	}
+	fmt.Fprintf(r.out, "JUnit report: %s\n", filepath.ToSlash(r.path))
+	return runErr
+}
+
+// writeJUnitReport replaces the file at path with the report.
+func writeJUnitReport(path string, report junitTestSuites) error {
+	data, err := xml.MarshalIndent(report, "", "  ")
+	if err != nil {
+		return err
+	}
+	data = append([]byte(xml.Header), data...)
+	data = append(data, '\n')
+	// A partly written file would be read as a broken report, so the file is replaced at once.
+	if err := atomicfile.Write(path, data, 0o644); err != nil {
+		return fmt.Errorf("failed to write JUnit report: %w", err)
+	}
+	return nil
+}

cmd/pipelines/test_junit_test.go

@@ -0,0 +1,76 @@
+package pipelines
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestJUnitCaseName(t *testing.T) {
+	tests := []struct {
+		testID, className, name string
+	}{
+		{"tests/test_orders.py::test_backfill", "tests.test_orders", "test_backfill"},
+		{"tests/test_orders.py::TestOrders::test_tax[a-b]", "tests.test_orders.TestOrders", "test_tax[a-b]"},
+		// A parameter ID may contain "::".
+		{"tests/test_a.py::test_cast[a::int]", "tests.test_a", "test_cast[a::int]"},
+		{"tests/test_a.py::TestCast::test_cast[a::int]", "tests.test_a.TestCast", "test_cast[a::int]"},
+		{"tests/test_orders.py", "", "tests.test_orders"},
+		{"/Workspace/Users/someone@example.com/tests/test_x.py::test_x", "Workspace.Users.someone@example.com.tests.test_x", "test_x"},
+		// A bracket in a folder name is part of the path, not a parameter ID.
+		{"tests/[weird]/test_a.py::test_x", "tests.[weird].test_a", "test_x"},
+	}
+	for _, tc := range tests {
+		className, name := junitCaseName(tc.testID)
+		assert.Equal(t, tc.className, className, tc.testID)
+		assert.Equal(t, tc.name, name, tc.testID)
+	}
+}
+
+func TestWriteJUnitReport(t *testing.T) {
+	ev := newTestEvents(t)
+	// Acceptance tests cover each outcome. This covers what the test server does not send:
+	// multi-line and truncated details, and text that XML must escape.
+	const project = "/Workspace/Users/someone@example.com/.bundle/orders/dev/files"
+	run := newTestRun()
+	for _, e := range []testPipelineEvent{
+		ev.testCase(project+"/tests/test_a.py::test_bad", "FAILED", "",
+			`"error": {"exceptions": [{"class_name": "AssertionError", "message": "assert 1 == 2\nwhere 1 = f()"}]}`,
+			`"truncation": {"truncated_fields": [{"field_name": "error"}]}`),
+		ev.testCase(project+"/tests/test_a.py::test_setup", "ERROR", "",
+			`"error": {"exceptions": [{"class_name": "RuntimeError", "message": "fixture <db> failed"}]}`),
+	} {
+		run.add(e)
+	}
+
+	path := filepath.Join(t.TempDir(), "report.xml")
+	require.NoError(t, writeJUnitReport(path, newJUnitReport("orders", "u1", project, run, "")))
+
+	data, err := os.ReadFile(path)
+	require.NoError(t, err)
+	assert.Equal(t, `<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+  <testsuite name="orders" tests="2" failures="1" errors="1" skipped="0">
+    <testcase classname="tests.test_a" name="test_bad">
+      <failure message="AssertionError: assert 1 == 2">AssertionError: assert 1 == 2&#xA;where 1 = f()&#xA;(details truncated)</failure>
+    </testcase>
+    <testcase classname="tests.test_a" name="test_setup">
+      <error message="RuntimeError: fixture &lt;db&gt; failed">RuntimeError: fixture &lt;db&gt; failed</error>
+    </testcase>
+  </testsuite>
+</testsuites>
+`, string(data))
+}
+
+func TestTestReportJoinsWriteErrors(t *testing.T) {
+	report := &testReport{path: filepath.Join(t.TempDir(), "missing", "report.xml"), out: io.Discard, suiteName: "orders"}
+	runErr := errors.New("update u1 is FAILED")
+	err := report.finish("u1", newTestRun(), runErr.Error(), runErr)
+	require.ErrorIs(t, err, runErr)
+	assert.ErrorContains(t, err, "failed to write JUnit report")
+}