diff --git a/notifications_utils/formatters.py b/notifications_utils/formatters.py index 652fa61de..446e9fb08 100644 --- a/notifications_utils/formatters.py +++ b/notifications_utils/formatters.py @@ -7,7 +7,7 @@ import bleach import mistune import smartypants -from flask import Markup +from flask import Markup, current_app from notifications_utils.sanitise_text import SanitiseSMS @@ -33,12 +33,20 @@ EN_CLOSE = r"\[\[/en\]\]" # matches [[/en]] RTL_OPEN = r"\[\[rtl\]\]" # matches [[rtl]] RTL_CLOSE = r"\[\[/rtl\]\]" # matches [[/rtl]] +CALLOUT_OPEN = r"\[\[callout\]\]" # matches [[callout]] +CALLOUT_CLOSE = r"\[\[/callout\]\]" # matches [[/callout]] +CTA_OPEN = r"\[\[cta\]\]" # matches [[cta]] +CTA_CLOSE = r"\[\[/cta\]\]" # matches [[/cta]] FR_OPEN_LITERAL = "[[fr]]" FR_CLOSE_LITERAL = "[[/fr]]" EN_OPEN_LITERAL = "[[en]]" EN_CLOSE_LITERAL = "[[/en]]" RTL_OPEN_LITERAL = "[[rtl]]" RTL_CLOSE_LITERAL = "[[/rtl]]" +CALLOUT_OPEN_LITERAL = "[[callout]]" +CALLOUT_CLOSE_LITERAL = "[[/callout]]" +CTA_OPEN_LITERAL = "[[cta]]" +CTA_CLOSE_LITERAL = "[[/cta]]" BR_TAG = r"" @@ -488,8 +496,42 @@ def double_emphasis(self, text): def emphasis(self, text): return f"{text}" + def table(self, header, body): + # If the feature flag is off, return the content with the tags unprocessed. This allows us to add table tags to + # content without them being rendered as tables until we're ready to turn the feature flag on + try: + if not current_app.config.get("FF_EMAIL_TABLES", False): + return "" + except RuntimeError: + return "" + return ( + '' + f"{header}" + f"{body}" + "
" + ) + + def table_row(self, content): + return f"{content}" + + def table_cell(self, content, **flags): + if flags.get("header"): + return ( + '{content}" + ) + + align = flags.get("align") + align_style = f"text-align: {align}; " if align else "" + return f'' f"{content}" + class NotifyPlainTextEmailMarkdownRenderer(NotifyEmailMarkdownRenderer): + _TABLE_CELL_SEPARATOR = "\u241f" + _TABLE_HEADER_PREFIX = "__TABLE_HEADER__:" + _TABLE_CELL_PREFIX = "__TABLE_CELL__:" + COLUMN_WIDTH = 65 def header(self, text, level, raw=None): @@ -570,6 +612,37 @@ def double_emphasis(self, text): def emphasis(self, text): return f"_{text}_" + def table(self, header, body): + # If the feature flag is off, return the content with the tags unprocessed. This allows us to add table tags to + # content without them being rendered as tables until we're ready to turn the feature flag on + try: + if not current_app.config.get("FF_EMAIL_TABLES", False): + return "" + except RuntimeError: + return "" + return "".join((self.linebreak() * 2, header, body.rstrip("\n"))) + + def table_row(self, content): + cells_with_markers = [cell for cell in content.split(self._TABLE_CELL_SEPARATOR) if cell] + if not cells_with_markers: + return "" + + is_header = all(cell.startswith(self._TABLE_HEADER_PREFIX) for cell in cells_with_markers) + cells = [ + cell.replace(self._TABLE_HEADER_PREFIX, "", 1).replace(self._TABLE_CELL_PREFIX, "", 1) for cell in cells_with_markers + ] + + row = f"| {' | '.join(cells)} |" + if is_header: + separator = f"| {' | '.join(['---'] * len(cells))} |" + return f"{row}\n{separator}\n" + + return f"{row}\n" + + def table_cell(self, content, **flags): + prefix = self._TABLE_HEADER_PREFIX if flags.get("header") else self._TABLE_CELL_PREFIX + return f"{prefix}{content}{self._TABLE_CELL_SEPARATOR}" + class NotifyEmailPreheaderMarkdownRenderer(NotifyPlainTextEmailMarkdownRenderer): def header(self, text, level, raw=None): @@ -685,6 +758,122 @@ def remove_rtl_divs(_content: str) -> str: return remove_tags(_content, RTL_OPEN, RTL_CLOSE) +def escape_callout_tags(_content: str) -> str: + """ + Escape callout tags into code tags in the content so mistune doesn't put them inside p tags. This makes it simple + to replace them afterwards, and avoids creating invalid HTML in the process + """ + + # check to ensure we have the same number of opening and closing tags before escaping tags + if _content.count(CALLOUT_OPEN_LITERAL) == _content.count(CALLOUT_CLOSE_LITERAL): + _content = _content.replace(CALLOUT_OPEN_LITERAL, f"\n```\n{CALLOUT_OPEN_LITERAL}\n```\n") + _content = _content.replace(CALLOUT_CLOSE_LITERAL, f"\n```\n{CALLOUT_CLOSE_LITERAL}\n```\n") + + return _content + + +def add_callout_divs(_content: str) -> str: + """ + Custom parser to add the callout divs. + + String replace callout tags in-place with styled div elements. + """ + # If the feature flag is off, return the content with the tags unprocessed. This allows us to add callout tags to + # content without them being rendered as callout divs until we're ready to turn the feature flag on. + try: + if not current_app.config.get("FF_EMAIL_CALLOUTS", False): + return _content + except RuntimeError: + return _content + + # check to ensure we have the same number of opening and closing tags before replacing tags + if _content.count(CALLOUT_OPEN_LITERAL) == _content.count(CALLOUT_CLOSE_LITERAL): + _content = _content.replace( + CALLOUT_OPEN_LITERAL, + '
', + ) + _content = _content.replace(CALLOUT_CLOSE_LITERAL, "
") + + return _content + + +def remove_callout_divs(_content: str) -> str: + """Remove the tags from content. This fn is for use in the email + preheader, since this is plain text not html""" + return remove_tags(_content, CALLOUT_OPEN, CALLOUT_CLOSE) + + +def escape_cta_tags(_content: str) -> str: + """ + Escape CTA tags into code tags in the content so mistune doesn't put them inside p tags. This makes it simple + to replace them afterwards, and avoids creating invalid HTML in the process + """ + + # check to ensure we have the same number of opening and closing tags before escaping tags + if _content.count(CTA_OPEN_LITERAL) == _content.count(CTA_CLOSE_LITERAL): + _content = _content.replace(CTA_OPEN_LITERAL, f"\n```\n{CTA_OPEN_LITERAL}\n```\n") + _content = _content.replace(CTA_CLOSE_LITERAL, f"\n```\n{CTA_CLOSE_LITERAL}\n```\n") + + return _content + + +def add_cta_buttons(_content: str) -> str: + """ + Custom parser to add CTA button divs. + + String replace CTA tags in-place with styled div elements, but only if the content + contains exactly one link ( tag). If zero or multiple links, leave tags unprocessed. + """ + # If the feature flag is off, return the content with the tags unprocessed. This allows us to add CTA tags to + # content without them being rendered as CTA buttons until we're ready to turn the feature flag on + try: + if not current_app.config.get("FF_EMAIL_CTA", False): + return _content + except RuntimeError: + return _content + + # check to ensure we have the same number of opening and closing tags before replacing tags + if _content.count(CTA_OPEN_LITERAL) == _content.count(CTA_CLOSE_LITERAL): + # Find all CTA blocks and validate each one + result = _content + import re as regex_module + + # Pattern to match CTA blocks + cta_pattern = regex_module.compile(r"\[\[cta\]\](.*?)\[\[/cta\]\]", regex_module.DOTALL) + + def replace_cta(match): + cta_content = match.group(1) + # Count tags in this CTA block + link_count = cta_content.count(" tag with !important + link_styled_content = regex_module.sub( + r' tag + link_styled_content = regex_module.sub(r'

{link_styled_content}' + else: + # Leave unprocessed if not exactly one link + return match.group(0) + + result = cta_pattern.sub(replace_cta, result) + return result + + return _content + + +def remove_cta_tags(_content: str) -> str: + """Remove the tags from content. This fn is for use in the email + preheader, since this is plain text not html""" + return remove_tags(_content, CTA_OPEN, CTA_CLOSE) + + def remove_tags(_content: str, *tags) -> str: """Remove the tags in parameters from content. diff --git a/notifications_utils/template.py b/notifications_utils/template.py index 03643862e..9dfd3fd7e 100644 --- a/notifications_utils/template.py +++ b/notifications_utils/template.py @@ -11,11 +11,15 @@ from notifications_utils.columns import Columns from notifications_utils.field import Field from notifications_utils.formatters import ( + add_callout_divs, + add_cta_buttons, add_language_divs, add_prefix, add_rtl_divs, add_trailing_newline, autolink_sms, + escape_callout_tags, + escape_cta_tags, escape_html, escape_lang_tags, escape_rtl_tags, @@ -28,6 +32,8 @@ notify_email_preheader_markdown, notify_letter_preview_markdown, notify_plain_text_email_markdown, + remove_callout_divs, + remove_cta_tags, remove_empty_lines, remove_language_divs, remove_nested_list_padding, @@ -423,6 +429,8 @@ def preheader(self): .then(notify_email_preheader_markdown) .then(remove_language_divs) .then(remove_rtl_divs) + .then(remove_callout_divs) + .then(remove_cta_tags) .then(do_nice_typography) .split() )[: self.PREHEADER_LENGTH_IN_CHARACTERS].strip() @@ -857,10 +865,14 @@ def get_html_email_body(template_content, template_values, redact_missing_person .then(add_trailing_newline) .then(escape_lang_tags) .then(escape_rtl_tags) + .then(escape_callout_tags) + .then(escape_cta_tags) .then(notify_email_markdown) .then(remove_nested_list_padding) .then(add_language_divs) .then(add_rtl_divs) + .then(add_callout_divs) + .then(add_cta_buttons) .then(do_nice_typography) ) diff --git a/tests/test_formatters.py b/tests/test_formatters.py index 7f862af5c..7aaeea7a2 100644 --- a/tests/test_formatters.py +++ b/tests/test_formatters.py @@ -557,11 +557,41 @@ def test_multiple_newlines_get_truncated(markdown_function, expected): assert markdown_function("before\n\n\n\n\n\nafter") == expected -@pytest.mark.parametrize( - "markdown_function", (notify_letter_preview_markdown, notify_email_markdown, notify_plain_text_email_markdown) -) -def test_table(markdown_function): - assert markdown_function("col | col\n" "----|----\n" "val | val\n") == ("") +def test_table_feature_off(): + markdown_input = "col | col\n" "----|----\n" "val | val\n" + + assert notify_letter_preview_markdown(markdown_input) == "" + + # Tables are gated behind FF_EMAIL_TABLES — off by default (no app context) + email_result = notify_email_markdown(markdown_input) + assert "' in html assert "RTL CONTENT" in html assert "<{}".format(extra_tag) in html + + +class TestCalloutTags: + def test_callout_tags_in_templates(self, app): + app.config["FF_EMAIL_CALLOUTS"] = True + content = "[[callout]]\nCallout content\n[[/callout]]" + html = get_html_email_body(content, {}) + assert "box-shadow: 0 1px 3px #0000000d" in html + assert "border: 1px solid #dcd6d6" in html + assert "Callout content" in html + + @pytest.mark.parametrize( + "nested_content", + [ + "[[callout]]\nCallout content\n[[/callout]]\n[[callout]]\nMore callout content\n[[/callout]]", + "[[callout]]\nCallout content with [[en]]\nEN content\n[[/en]]\n[[/callout]]", + "[[callout]]\nCallout content with [[rtl]]\nRTL content\n[[/rtl]]\n[[/callout]]", + "[[rtl]]\n[[callout]]\nCallout in RTL\n[[/callout]]\n[[/rtl]]", + ], + ) + def test_callout_tags_in_templates_nested_content(self, app, nested_content: str): + app.config["FF_EMAIL_CALLOUTS"] = True + html = get_html_email_body(nested_content, {}) + assert "box-shadow: 0 1px 3px #0000000d" in html + assert "Callout" in html + + @pytest.mark.parametrize( + "bad_content", + [ + "[[callout]\nCallout content\n[[/callout]]", # missing bracket + "[[CALLOUT]]\nCallout content\n[[/CALLOUT]]", # tags not lowercase + "[[callout]]\nCallout content\n", # tag missing + "Callout content\n[[/callout]]", # tag missing + "((callout))\nCallout content\n((/callout))", # wrong brackets + ], + ) + def test_callout_tags_in_templates_bad_content(self, bad_content: str): + html = get_html_email_body(bad_content, {}) + assert "box-shadow" not in html + + @pytest.mark.parametrize( + "mixed_content", + [ + "[[callout]]\nCallout content\n[[/callout]]\nRegular content", + "Regular content\n[[callout]]\nCallout content\n[[/callout]]", + ], + ) + def test_callout_tags_in_templates_mixed_content(self, app, mixed_content: str): + app.config["FF_EMAIL_CALLOUTS"] = True + html = get_html_email_body(mixed_content, {}) + assert "box-shadow: 0 1px 3px #0000000d" in html + assert "Callout content" in html + assert "Regular content" in html + + @pytest.mark.parametrize( + "content, extra_tag", + [ + ("[[callout]]# CALLOUT CONTENT[[/callout]]", "h2"), + ("[[callout]]## CALLOUT CONTENT[[/callout]]", "h3"), + ("[[callout]]\n- CALLOUT CONTENT 1\n- item 2\n[[/callout]]", "ul"), + ("[[callout]]\n1. CALLOUT CONTENT 1\n1. item 2\n[[/callout]]", "ol"), + ("[[callout]]**CALLOUT CONTENT**[[/callout]]", "strong"), + ("[[callout]]_CALLOUT CONTENT_[[/callout]]", "em"), + ("[[callout]]---\nCALLOUT CONTENT[[/callout]]", "hr"), + ("[[callout]]^CALLOUT CONTENT[[/callout]]", "blockquote"), + ("[[callout]]CALLOUT CONTENT now at https://www.canada.ca[[/callout]]", "a"), + ("[[callout]][CALLOUT CONTENT](https://www.canada.ca/sign-in)[[/callout]]", "a"), + ("[[callout]][[en]]CALLOUT CONTENT[[/en]][[/callout]]", 'div lang="en-ca"'), + ("[[callout]][[fr]]CALLOUT CONTENT[[/fr]][[/callout]]", 'div lang="fr-ca"'), + ("[[callout]][[rtl]]CALLOUT CONTENT[[/rtl]][[/callout]]", 'div dir="rtl"'), + ], + ids=[ + "heading_1", + "heading_2", + "list_unordered", + "list_ordered", + "bold", + "italic", + "hr", + "blockquote", + "link", + "link_with_text", + "nested_lang_tags_en", + "nested_lang_tags_fr", + "nested_rtl_tags", + ], + ) + def test_callout_tags_work_with_other_features(self, app, content: str, extra_tag: str): + app.config["FF_EMAIL_CALLOUTS"] = True + html = get_html_email_body(content, {}) + assert "box-shadow: 0 1px 3px #0000000d" in html + assert "CALLOUT CONTENT" in html + assert "<{}".format(extra_tag) in html + + def test_callout_not_rendered_when_feature_off(self, app): + app.config["FF_EMAIL_CALLOUTS"] = False + content = "[[callout]]\nCallout content\n[[/callout]]" + html = get_html_email_body(content, {}) + assert "box-shadow" not in html + assert "Callout content" in html # text still present, just unstyled + + +class TestCTATags: + def test_cta_with_single_link(self, app): + app.config["FF_EMAIL_CTA"] = True + content = "[[cta]][Sign up now](https://example.com)[[/cta]]" + html = get_html_email_body(content, {}) + assert "background: #ffbf47" in html + assert "Sign up now" in html + assert "https://example.com" in html + + def test_cta_with_no_links_unprocessed(self, app): + app.config["FF_EMAIL_CTA"] = True + content = "[[cta]]Click here[[/cta]]" + html = get_html_email_body(content, {}) + # CTA tags should remain unprocessed (no link inside) + assert "background: #ffbf47" not in html + assert "[[cta]]" in html + + def test_cta_with_multiple_links_unprocessed(self, app): + app.config["FF_EMAIL_CTA"] = True + content = "[[cta]][Link 1](https://example.com) and [Link 2](https://example.com)[[/cta]]" + html = get_html_email_body(content, {}) + # CTA tags should remain unprocessed (multiple links) + assert "background: #ffbf47" not in html + assert "[[cta]]" in html + + def test_cta_with_link_and_text(self, app): + app.config["FF_EMAIL_CTA"] = True + content = "[[cta]]Click **here** to [sign up](https://example.com)[[/cta]]" + html = get_html_email_body(content, {}) + assert "background: #ffbf47" in html + assert "here" in html + assert "https://example.com" in html + + def test_multiple_valid_ctas(self, app): + app.config["FF_EMAIL_CTA"] = True + content = "[[cta]][First](https://example.com)[[/cta]]\n\n[[cta]][Second](https://example.com)[[/cta]]" + html = get_html_email_body(content, {}) + # Both should be processed + assert html.count("background: #ffbf47") == 2 + + def test_cta_with_language_tags(self, app): + app.config["FF_EMAIL_CTA"] = True + content = "[[cta]][[en]][Sign up](https://example.com)[[/en]][[/cta]]" + html = get_html_email_body(content, {}) + assert "background: #ffbf47" in html + assert 'lang="en-ca"' in html + assert "https://example.com" in html + + def test_cta_with_rtl(self, app): + app.config["FF_EMAIL_CTA"] = True + content = "[[cta]][[rtl]][Click](https://example.com)[[/rtl]][[/cta]]" + html = get_html_email_body(content, {}) + assert "background: #ffbf47" in html + assert 'dir="rtl"' in html + + def test_cta_inside_callout(self, app): + app.config["FF_EMAIL_CALLOUTS"] = True + app.config["FF_EMAIL_CTA"] = True + content = "[[callout]]\n[[cta]][Learn more](https://example.com)[[/cta]]\n[[/callout]]" + html = get_html_email_body(content, {}) + assert "box-shadow" in html # callout styling + assert "background: #ffbf47" in html # CTA styling + + @pytest.mark.parametrize( + "bad_content", + [ + "[[cta]\n[Link](https://example.com)\n[[/cta]]", # missing bracket + "[[CTA]][Link](https://example.com)[[/CTA]]", # tags not lowercase + "[[cta]][Link](https://example.com)\n", # tag missing + "[Link](https://example.com)\n[[/cta]]", # tag missing + "((cta))[Link](https://example.com)((/cta))", # wrong brackets + ], + ) + def test_cta_malformed_tags_unprocessed(self, bad_content: str): + html = get_html_email_body(bad_content, {}) + # Malformed tags should not be processed + assert "background: #ffbf47" not in html + + def test_cta_not_rendered_when_feature_off(self, app): + app.config["FF_EMAIL_CTA"] = False + content = "[[cta]][Sign up now](https://example.com)[[/cta]]" + html = get_html_email_body(content, {}) + assert "background: #ffbf47" not in html + assert "Sign up now" in html # link text still present, just not styled + + +class TestTableTags: + def test_markdown_tables_not_rendered_when_feature_off(self, app): + app.config["FF_EMAIL_TABLES"] = False + content = "| Syntax | Description |\n| ----------- | ----------- |\n| Header | Title |\n| Paragraph | Text |" + html = get_html_email_body(content, {}) + + assert "