Skip to content

Commit 1362d87

Browse files
OEvgenyEugene Olonov
OEvgeny
and
Eugene Olonov
authored
Read Adaptive Card text in the live region as the card shows it (#5929)
* Move setting the Adaptive Cards Markdown engine into setMarkdownEngine * Read Adaptive Card text in the live region as the card shows it Text blocks, rich text blocks, and facts are rendered with Adaptive Cards and the same Markdown engine as the visible card, so Markdown syntax and {{DATE()}}/{{TIME()}} are no longer read as written. * Let htmlTextAlternatives read any HTML node and export it as internal * Read Adaptive Card text in the live region with htmlTextAlternatives() Image alt text in Markdown is read, and HTML blocks without whitespace between them are no longer read as one word. * Fix test comments: the live region is above the card in the snapshots * Read Adaptive Card text as written if rendering it throws * Move getFactText to module level * Comment on setting the shared Markdown engine while rendering * Compare the Markdown test with the text of the visible card * Collapse whitespace of the raw text fallback too * Add PR number to changelog * Move the changelog entry to .changelog/ --------- Co-authored-by: Eugene Olonov <eugeneolonov@microsoft.com>
1 parent 3d4881a commit 1362d87

12 files changed

Lines changed: 355 additions & 29 deletions

File tree

‎.changelog/5929-fix-liveregion-card-text-format.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
### Fixed
2+
3+
- Fixed screen readers reading Adaptive Card text with its Markdown and `{{DATE()}}` syntax, in PR [#5929](https://github.com/microsoft/BotFramework-WebChat/pull/5929), by [@OEvgeny](https://github.com/OEvgeny)

‎__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
<!doctype html>
2+
<html lang="en-US">
3+
<head>
4+
<link href="/assets/index.css" rel="stylesheet" type="text/css" />
5+
<!-- Shows the live region, so the snapshot shows the copy sent to screen readers. -->
6+
<link href="/assets/accessibility.liveRegionAttachment.css" rel="stylesheet" type="text/css" />
7+
</head>
8+
<body>
9+
<main id="webchat"></main>
10+
<script type="importmap">
11+
{
12+
"imports": {
13+
"botframework-webchat": "/__dist__/packages/bundle/static/botframework-webchat.js",
14+
"botframework-webchat/": "/__dist__/packages/bundle/static/botframework-webchat/",
15+
"react": "/__dist__/packages/bundle/static/react.js",
16+
"react-dom": "/__dist__/packages/bundle/static/react-dom.js",
17+
"react-dom/client": "/__dist__/packages/bundle/static/react-dom/client.js"
18+
}
19+
}
20+
</script>
21+
<script type="module">
22+
import '/test-harness.mjs';
23+
import '/test-page-object.mjs';
24+
25+
import { createStoreWithOptions, ReactWebChat } from 'botframework-webchat';
26+
import { createElement } from 'react';
27+
import { createRoot } from 'react-dom/client';
28+
29+
// TODO: This is for `createDirectLineEmulator` only, should find ways to eliminate this line.
30+
window.WebChat = { createStoreWithOptions };
31+
32+
run(async function () {
33+
const { directLine, store } = testHelpers.createDirectLineEmulator();
34+
35+
// GIVEN: Web Chat keeps the live region copy of new messages, so it can be checked and seen in the snapshot.
36+
createRoot(document.getElementById('webchat')).render(
37+
createElement(ReactWebChat, {
38+
directLine,
39+
store,
40+
styleOptions: { hideSendBox: true, internalLiveRegionFadeAfter: 60000 }
41+
})
42+
);
43+
44+
await pageConditions.uiConnected();
45+
46+
// WHEN: The bot sends an Adaptive Card with date and time functions, and without "speak".
47+
await directLine.emulateIncomingActivity({
48+
attachments: [
49+
{
50+
content: {
51+
$schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
52+
body: [
53+
{
54+
text: 'Arrives {{DATE(2026-10-20T15:00:00Z, SHORT)}} at {{TIME(2026-10-20T15:00:00Z)}}',
55+
type: 'TextBlock',
56+
wrap: true
57+
}
58+
],
59+
type: 'AdaptiveCard',
60+
version: '1.5'
61+
},
62+
contentType: 'application/vnd.microsoft.card.adaptive'
63+
}
64+
],
65+
from: { role: 'bot' },
66+
type: 'message'
67+
});
68+
69+
await pageConditions.numActivitiesShown(1);
70+
await pageConditions.liveRegionStabilized();
71+
72+
// Adaptive Cards formats the date and time with the browser locale and time zone, so read them from the card as shown.
73+
const cardText = pageElements.activities()[0].querySelector('.ac-textBlock').textContent.trim();
74+
75+
// THEN: The card shows the formatted date and time, not the functions.
76+
expect(cardText).toEqual(expect.stringMatching(/^Arrives \S.* at \S/u));
77+
expect(cardText).not.toContain('{{');
78+
79+
// THEN: The live region copy reads the same formatted date and time as the card.
80+
expect(
81+
testHelpers
82+
.getAllTextContents(pageElements.liveRegion().querySelector('.webchat__live-region-activity'))
83+
.map(text => text.trim())
84+
).toEqual(['Bot said:', `A card: ${cardText}`]);
85+
86+
// THEN: The card and the live region above show the same formatted date and time.
87+
await host.snapshot('local');
88+
});
89+
</script>
90+
</body>
91+
</html>

‎__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.dateTime.html.snap-1.png‎

11.2 KB
Loading

‎__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
<!doctype html>
2+
<html lang="en-US">
3+
<head>
4+
<link href="/assets/index.css" rel="stylesheet" type="text/css" />
5+
<!-- Shows the live region, so the snapshot shows the copy sent to screen readers. -->
6+
<link href="/assets/accessibility.liveRegionAttachment.css" rel="stylesheet" type="text/css" />
7+
</head>
8+
<body>
9+
<main id="webchat"></main>
10+
<script type="importmap">
11+
{
12+
"imports": {
13+
"@testduet/wait-for": "https://esm.sh/@testduet/wait-for",
14+
"botframework-webchat": "/__dist__/packages/bundle/static/botframework-webchat.js",
15+
"botframework-webchat/": "/__dist__/packages/bundle/static/botframework-webchat/",
16+
"react": "/__dist__/packages/bundle/static/react.js",
17+
"react-dom": "/__dist__/packages/bundle/static/react-dom.js",
18+
"react-dom/client": "/__dist__/packages/bundle/static/react-dom/client.js"
19+
}
20+
}
21+
</script>
22+
<script type="module">
23+
import '/test-harness.mjs';
24+
import '/test-page-object.mjs';
25+
26+
import { waitFor } from '@testduet/wait-for';
27+
import { createStoreWithOptions, ReactWebChat } from 'botframework-webchat';
28+
import { createElement } from 'react';
29+
import { createRoot } from 'react-dom/client';
30+
31+
// TODO: This is for `createDirectLineEmulator` only, should find ways to eliminate this line.
32+
window.WebChat = { createStoreWithOptions };
33+
34+
run(async function () {
35+
const { directLine, store } = testHelpers.createDirectLineEmulator();
36+
37+
// GIVEN: Web Chat keeps the live region copy of new messages, so it can be checked and seen in the snapshot.
38+
createRoot(document.getElementById('webchat')).render(
39+
createElement(ReactWebChat, {
40+
directLine,
41+
store,
42+
styleOptions: { hideSendBox: true, internalLiveRegionFadeAfter: 60000 }
43+
})
44+
);
45+
46+
await pageConditions.uiConnected();
47+
48+
// WHEN: The bot sends an Adaptive Card with an image in Markdown and HTML blocks without whitespace between them, and without "speak".
49+
await directLine.emulateIncomingActivity({
50+
attachments: [
51+
{
52+
content: {
53+
$schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
54+
body: [
55+
{
56+
text: '![Bot avatar](/assets/bot-avatar.jpg) Weather today<div>Sunny</div><p>Umbrella: no</p>',
57+
type: 'TextBlock',
58+
wrap: true
59+
}
60+
],
61+
type: 'AdaptiveCard',
62+
version: '1.5'
63+
},
64+
contentType: 'application/vnd.microsoft.card.adaptive'
65+
}
66+
],
67+
from: { role: 'bot' },
68+
type: 'message'
69+
});
70+
71+
await pageConditions.numActivitiesShown(1);
72+
await pageConditions.liveRegionStabilized();
73+
74+
const textBlock = pageElements.activities()[0].querySelector('.ac-textBlock');
75+
76+
// THEN: The card shows the image with its alt text, and each block on its own line.
77+
await waitFor(() => expect(textBlock.querySelector('img[alt="Bot avatar"]').complete).toBe(true));
78+
expect(textBlock.innerText.trim().split(/\s*\n\s*/u)).toEqual(['Weather today', 'Sunny', 'Umbrella: no']);
79+
80+
// THEN: The live region copy reads the image alt text, and separates the blocks.
81+
expect(
82+
testHelpers
83+
.getAllTextContents(pageElements.liveRegion().querySelector('.webchat__live-region-activity'))
84+
.map(text => text.trim())
85+
).toEqual(['Bot said:', 'A card: Bot avatar Weather today Sunny Umbrella: no']);
86+
87+
// THEN: The card shows the image and three lines; the live region above reads the alt text and the three lines.
88+
await host.snapshot('local');
89+
});
90+
</script>
91+
</body>
92+
</html>

‎__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.imageAndBlocks.html.snap-1.png‎

24.6 KB
Loading

‎__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
<!doctype html>
2+
<html lang="en-US">
3+
<head>
4+
<link href="/assets/index.css" rel="stylesheet" type="text/css" />
5+
<!-- Shows the live region, so the snapshot shows the copy sent to screen readers. -->
6+
<link href="/assets/accessibility.liveRegionAttachment.css" rel="stylesheet" type="text/css" />
7+
</head>
8+
<body>
9+
<main id="webchat"></main>
10+
<script type="importmap">
11+
{
12+
"imports": {
13+
"botframework-webchat": "/__dist__/packages/bundle/static/botframework-webchat.js",
14+
"botframework-webchat/": "/__dist__/packages/bundle/static/botframework-webchat/",
15+
"react": "/__dist__/packages/bundle/static/react.js",
16+
"react-dom": "/__dist__/packages/bundle/static/react-dom.js",
17+
"react-dom/client": "/__dist__/packages/bundle/static/react-dom/client.js"
18+
}
19+
}
20+
</script>
21+
<script type="module">
22+
import '/test-harness.mjs';
23+
import '/test-page-object.mjs';
24+
25+
import { createStoreWithOptions, ReactWebChat } from 'botframework-webchat';
26+
import { createElement } from 'react';
27+
import { createRoot } from 'react-dom/client';
28+
29+
// TODO: This is for `createDirectLineEmulator` only, should find ways to eliminate this line.
30+
window.WebChat = { createStoreWithOptions };
31+
32+
run(async function () {
33+
const { directLine, store } = testHelpers.createDirectLineEmulator();
34+
35+
// GIVEN: Web Chat keeps the live region copy of new messages, so it can be checked and seen in the snapshot.
36+
createRoot(document.getElementById('webchat')).render(
37+
createElement(ReactWebChat, {
38+
directLine,
39+
store,
40+
styleOptions: { hideSendBox: true, internalLiveRegionFadeAfter: 60000 }
41+
})
42+
);
43+
44+
await pageConditions.uiConnected();
45+
46+
// WHEN: The bot sends an Adaptive Card with a Markdown text block (bold, a link, and a bullet list), and without "speak".
47+
await directLine.emulateIncomingActivity({
48+
attachments: [
49+
{
50+
content: {
51+
$schema: 'http://adaptivecards.io/schemas/adaptive-card.json',
52+
body: [
53+
{
54+
text: 'Today is **sunny**, see [docs](https://example.com).\n\n- Umbrella: no\n- Sunscreen: yes',
55+
type: 'TextBlock',
56+
wrap: true
57+
}
58+
],
59+
type: 'AdaptiveCard',
60+
version: '1.5'
61+
},
62+
contentType: 'application/vnd.microsoft.card.adaptive'
63+
}
64+
],
65+
from: { role: 'bot' },
66+
type: 'message'
67+
});
68+
69+
await pageConditions.numActivitiesShown(1);
70+
await pageConditions.liveRegionStabilized();
71+
72+
// The card shows the list items on separate lines, the live region reads them as one line.
73+
const cardText = pageElements
74+
.activities()[0]
75+
.querySelector('.ac-textBlock')
76+
.textContent.replace(/\s+/gu, ' ')
77+
.trim();
78+
79+
// THEN: The card shows the text without Markdown syntax ("**", "[", "](", "- ").
80+
expect(cardText).toEqual(
81+
expect.stringMatching(/^Today is sunny, see .*docs.*\. Umbrella: no Sunscreen: yes$/u)
82+
);
83+
84+
// THEN: The live region copy reads the same text as the card.
85+
expect(
86+
testHelpers
87+
.getAllTextContents(pageElements.liveRegion().querySelector('.webchat__live-region-activity'))
88+
.map(text => text.trim())
89+
).toEqual(['Bot said:', `A card: ${cardText}`]);
90+
91+
// THEN: The card shows bold "sunny", a "docs" link and a bullet list; the live region above shows the same words without Markdown syntax.
92+
await host.snapshot('local');
93+
});
94+
</script>
95+
</body>
96+
</html>

‎__tests__/html2/accessibility/liveRegion/attachment/adaptiveCard.markdown.html.snap-1.png‎

20.5 KB
Loading

‎packages/bundle/src/adaptiveCards/Attachment/private/renderAdaptiveCard.ts‎

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
1-
import type {
2-
AdaptiveCard,
3-
GlobalSettings as GlobalSettingsType,
4-
HostConfig as HostConfigType,
5-
IMarkdownProcessingResult
6-
} from 'adaptivecards';
1+
import type { AdaptiveCard, GlobalSettings as GlobalSettingsType, HostConfig as HostConfigType } from 'adaptivecards';
2+
3+
import setMarkdownEngine from './setMarkdownEngine';
74

85
/**
96
* Returns `true`, if the object is a plain object and not a class, otherwise, `false`.
@@ -29,19 +26,7 @@ export default function renderAdaptiveCard(
2926
setTabIndexAtCardRoot: boolean;
3027
}
3128
): { element: HTMLElement } | { errors: Error[] } {
32-
// Currently, the only way to set the Markdown engine is to set it thru static member of AdaptiveCard class
33-
34-
// TODO: [P3] Checks if we could make the "renderMarkdownAsHTML" per card
35-
// This could be limitations from Adaptive Cards package (not supported as of 1.2.5)
36-
// Because there could be timing difference between .parse and .render, we could be using wrong Markdown engine
37-
38-
// "onProcessMarkdown" is a static function but we are trying to scope it to the current object instead.
39-
adaptiveCard.constructor['onProcessMarkdown'] = (text: string, result: IMarkdownProcessingResult) => {
40-
if (renderMarkdownAsHTML) {
41-
result.outputHtml = renderMarkdownAsHTML(text);
42-
result.didProcess = true;
43-
}
44-
};
29+
setMarkdownEngine(adaptiveCard, renderMarkdownAsHTML);
4530

4631
if (adaptiveCardsHostConfig) {
4732
adaptiveCard.hostConfig = isPlainObject(adaptiveCardsHostConfig)

‎packages/bundle/src/adaptiveCards/Attachment/private/setMarkdownEngine.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import type { AdaptiveCard, IMarkdownProcessingResult } from 'adaptivecards';
2+
3+
export default function setMarkdownEngine(
4+
adaptiveCard: AdaptiveCard,
5+
renderMarkdownAsHTML: ((markdown: string) => string) | undefined
6+
): void {
7+
// Currently, the only way to set the Markdown engine is to set it thru static member of AdaptiveCard class
8+
9+
// TODO: [P3] Checks if we could make the "renderMarkdownAsHTML" per card
10+
// This could be limitations from Adaptive Cards package (not supported as of 1.2.5)
11+
// Because there could be timing difference between .parse and .render, we could be using wrong Markdown engine
12+
13+
// "onProcessMarkdown" is a static function but we are trying to scope it to the current object instead.
14+
adaptiveCard.constructor['onProcessMarkdown'] = (text: string, result: IMarkdownProcessingResult) => {
15+
if (renderMarkdownAsHTML) {
16+
result.outputHtml = renderMarkdownAsHTML(text);
17+
result.didProcess = true;
18+
}
19+
};
20+
}

0 commit comments

Comments
 (0)