normalize newlines before doing the character count for sms (#407)

Author: smcmurtry

.github/actions/waffles/requirements.txt

@@ -2,4 +2,4 @@ docopt==0.6.2
 Flask==2.3.3
 markupsafe==2.1.5
 setuptools==78.1.1 # required for distutils in Python 3.12
-git+https://github.com/cds-snc/notifier-utils.git@53.2.20#egg=notifications-utils
+git+https://github.com/cds-snc/notifier-utils.git@53.2.21#egg=notifications-utils

notifications_utils/template.py

@@ -211,11 +211,14 @@ def _encoded_content(self):
         When values are set, placeholders are already replaced via __str__. When no values are
         set, placeholder syntax is stripped before encoding so that placeholder names don't
         inflate the character count or skew Unicode detection.
+
+        normalise_newlines is applied in both paths so that CRLF sequences (\\r\\n) submitted by
+        browsers are counted as a single newline unit, matching what is actually transmitted.
         """
         if self._values:
             # we always want to call SMSMessageTemplate.__str__ regardless of subclass, to avoid any html formatting
             return SMSMessageTemplate.__str__(self)
-        return sms_encode(add_prefix(Field.placeholder_pattern.sub("", self.content.strip()), self.prefix))
+        return normalise_newlines(sms_encode(add_prefix(Field.placeholder_pattern.sub("", self.content.strip()), self.prefix)))
 
     @property
     def content_count(self):

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "notifications-utils"
-version = "53.2.20"
+version = "53.2.21"
 description = "Shared python code for Notification - Provides logging utils etc."
 authors = ["Canadian Digital Service"]
 license = "MIT license"

tests/test_base_template.py

@@ -78,8 +78,15 @@ def test_extracting_placeholders(template_content, template_subject, expected):
         # should be replaced with a ?
         ("深", None, 1, 1),
         ("'First line.\n", None, 12, 12),
-        ("\t\n\r", None, 0, 0),
-        # variables do not count towards the character count for sms, since they will be replaced
+        ("\t\n\r", None, 0, 0),  # CRLF newlines (\r\n) from browser form submissions must be normalised to \n
+        # before counting, so each line break costs exactly 1 unit (not 2).
+        ("Hello\r\nWorld", None, 11, 11),
+        (
+            "Line1\r\nLine2\r\nLine3",
+            None,
+            17,
+            17,
+        ),  # variables do not count towards the character count for sms, since they will be replaced
         ("((placeholder))", None, 0, 3),
         ("((placeholder))", "Service name", 14, 17),
         ("Foo", "((placeholder))", 20, 20),  # placeholder doesn’t work in service name