diff --git a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java index 44071872292c..6c5073733aa2 100644 --- a/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java +++ b/dspace-oai/src/main/java/org/dspace/xoai/util/ItemUtils.java @@ -17,11 +17,11 @@ import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; import com.lyncode.xoai.dataprovider.xml.xoai.Element; import com.lyncode.xoai.dataprovider.xml.xoai.Metadata; import com.lyncode.xoai.util.Base64Utils; -import org.apache.commons.text.StringEscapeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.dspace.app.util.factory.UtilServiceFactory; @@ -256,8 +256,35 @@ private static Element createBundlesElement(Context context, Item item, AtomicBo } /** - * Sanitizes a string to remove characters that are invalid - * in XML 1.0 using the Apache Commons Text library. + * Matches everything XML 1.0 forbids outright, in three alternations: + *
    + *
  1. C0 controls other than tab, LF and CR, plus the non-characters U+FFFE and U+FFFF;
  2. + *
  3. a high surrogate not followed by a low surrogate;
  4. + *
  5. a low surrogate not preceded by a high surrogate.
  6. + *
+ * See https://www.w3.org/TR/xml/#charsets. Unpaired surrogates matter in practice: a truncated + * 4-byte character in ingested metadata makes the StAX writer throw "Broken surrogate pair", and + * because XOAI.index() catches that per item the record is silently dropped from the OAI index. + */ + private static final Pattern INVALID_XML10_CHARS = Pattern.compile( + "[\\x00-\\x08\\x0B\\x0C\\x0E-\\x1F\\uFFFE\\uFFFF]" + + "|[\\uD800-\\uDBFF](?![\\uDC00-\\uDFFF])" + + "|(? + * NOTE: this deliberately REMOVES illegal characters rather than escaping the string. The value + * returned here is handed to the XOAI serializer, which performs XML escaping itself (every text + * event goes through {@code XMLStreamWriter.writeCharacters}), so escaping here as well would + * double-escape every value containing &, <, >, " or ' — a harvester would then read the + * literal text "&lt;" instead of a "<" character. That silently corrupts every OAI format + * built on the xoai document, including the cmdi and olac formats CLARIN/LINDAT is aggregated + * through. + *

+ * The removal set must stay equivalent to what {@code StringEscapeUtils.escapeXml10} removed — + * notably including unpaired surrogates — otherwise items carrying them fail to serialize and + * drop out of the OAI index entirely. * @param value The string to sanitize. * @return A sanitized string, or null if the input was null. */ @@ -265,7 +292,7 @@ private static String sanitize(String value) { if (value == null) { return null; } - return StringEscapeUtils.escapeXml10(value); + return INVALID_XML10_CHARS.matcher(value).replaceAll(""); } /**