Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/desktop-element-screenshots.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
'@fluentui-react-native/desktop-driver': patch
---

Route WebdriverIO element.takeScreenshot() through the element screenshot endpoint instead of capturing the entire window.
5 changes: 5 additions & 0 deletions lage.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,11 @@ const config = {
clean: {
cache: false,
},
// The root script already cleans TypeScript output before invoking Lage.
'@fluentui-react-native/root#clean': {
type: 'noop',
cache: false,
},
'check:spec-contracts': {
cache: false,
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,15 @@ Use `@fluentui-react-native/desktop-test` for an application-owned
configuration and normal WDIO/Jasmine suites. It preserves TypeScript hooks,
Jasmine matchers, grouped spec arrays, retries, and native browser ownership.
`installDesktopWebdriverCommands(browser, url)` binds an existing WDIO-managed
browser, including native `isDisplayed`, `isFocused`, and delayed-render waits.
browser, including native `isDisplayed`, `isFocused`, element screenshots, and delayed-render waits.
No Appium capability spoofing or second session is required.

With these commands installed, `element.takeScreenshot()` returns a base64 PNG
cropped to the element, just like `browser.takeElementScreenshot(element.elementId)`.
`element.saveScreenshot(path)` saves that element image, while
`browser.takeScreenshot()` still captures the current window. Unsupported
element capture fails explicitly; it never falls back to a window screenshot.

Use `connectDesktopWebdriver` when a test owns service connection and session
lifecycle:

Expand Down
44 changes: 41 additions & 3 deletions packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,42 @@ import os from 'node:os';
import path from 'node:path';

import type { DesktopStoryManifest } from '../storybook.js';
import { createDesktopDriverStoryHarness } from '../testing/protocolHarness.js';
import { createDesktopDriverStoryHarness, createDesktopDriverTestHarness } from '../testing/protocolHarness.js';

describe('sanctioned WebdriverIO API', () => {
test.each([true, false])(
'routes element screenshots without falling back to window capture (supported: %s)',
async (elementScreenshot) => {
const harness = await createDesktopDriverTestHarness({ features: { elementScreenshot } });
const captureElement = jest.spyOn(harness.host, 'captureElement');
const captureWindow = jest.spyOn(harness.host, 'captureWindow');
const windowImage = await harness.host.captureWindow('window-1');
const elementImage = Buffer.from('element-image');
captureWindow.mockClear();
if (elementScreenshot) {
captureElement.mockResolvedValue({ ...windowImage, data: elementImage });
}
const artifactsRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'desktop-driver-screenshots-'));
try {
const response = await runContract(harness.server.url, harness.target.id, artifactsRoot, 'screenshots.contract.cjs');
for (const mode of ['connected', 'attached', 'managed']) {
expect(response[mode]).toEqual({
window: Buffer.from(windowImage.data).toString('base64'),
element: elementScreenshot ? elementImage.toString('base64') : 'unsupported',
savedElement: elementScreenshot ? elementImage.toString('base64') : 'unsupported',
});
}
expect(captureElement.mock.calls.length).toBeGreaterThanOrEqual(6);
expect(captureElement.mock.calls.every(([id]) => id === 'button')).toBe(true);
expect(captureWindow).toHaveBeenCalledTimes(3);
} finally {
await harness.close();
fs.rmSync(artifactsRoot, { force: true, recursive: true });
}
},
60_000,
);

test('lists and runs an authored plan through registered browser commands', async () => {
const manifest: DesktopStoryManifest = {
endpoint: 'windows',
Expand Down Expand Up @@ -68,9 +101,14 @@ describe('sanctioned WebdriverIO API', () => {
}, 60_000);
});

function runContract(url: string, targetId: string, artifactsRoot: string): Promise<Record<string, unknown>> {
function runContract(
url: string,
targetId: string,
artifactsRoot: string,
fixture = 'wdioRunner.contract.cjs',
): Promise<Record<string, unknown>> {
return new Promise((resolve, reject) => {
const child = spawn(process.execPath, [path.join(__dirname, 'wdioRunner.contract.cjs'), url, targetId, artifactsRoot], {
const child = spawn(process.execPath, [path.join(__dirname, fixture), url, targetId, artifactsRoot], {
stdio: ['ignore', 'pipe', 'pipe'],
});
const stdout: Buffer[] = [];
Expand Down
8 changes: 8 additions & 0 deletions packages/agentic/desktop-driver/src/wdio/DesktopWebdriver.ts
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,14 @@ export function installDesktopWebdriverCommands(browser: WebdriverIO.Browser, ur
boundSessions.set(browser, desktop);

if (!installedBrowsers.has(browser)) {
// WDIO otherwise inherits the session-wide screenshot command on elements.
browser.addCommand(
'takeScreenshot',
async function (this: WebdriverIO.Element) {
return this.takeElementScreenshot(this.elementId);
},
{ attachToElement: true },
);
browser.overwriteCommand(
'isDisplayed',
async function (this: WebdriverIO.Element) {
Expand Down
53 changes: 53 additions & 0 deletions packages/agentic/desktop-driver/src/wdio/screenshots.contract.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
const assert = require('node:assert/strict');
const fs = require('node:fs/promises');
const path = require('node:path');
const { pathToFileURL } = require('node:url');
const { attach } = require('webdriverio');

async function capture(browser, artifactsRoot, mode) {
const element = await browser.$('~button-primary');
const window = await browser.takeScreenshot();
const filename = path.join(artifactsRoot, `${mode}.png`);
if (!browser.capabilities['furn:features'].elementScreenshot) {
await assert.rejects(() => element.takeScreenshot(), /does not support element screenshots/);
await assert.rejects(() => element.saveScreenshot(filename), /does not support element screenshots/);
return { window, element: 'unsupported', savedElement: 'unsupported' };
}
const image = await element.takeScreenshot();
const saved = await element.saveScreenshot(filename);
assert.deepEqual(await fs.readFile(filename), saved);
return { window, element: image, savedElement: saved.toString('base64') };
}

async function main() {
const moduleUrl = pathToFileURL(path.resolve(__dirname, '..', '..', 'lib', 'wdio', 'index.js')).href;
const { attachDesktopWebdriver, connectDesktopWebdriver, installDesktopWebdriverCommands } = await import(moduleUrl);
const [url, targetId, artifactsRoot] = process.argv.slice(2);
const desktop = await connectDesktopWebdriver({ platformName: 'windows', targetId, url });
try {
const connected = await capture(desktop.browser, artifactsRoot, 'connected');
const attachment = { url, sessionId: desktop.browser.sessionId, capabilities: desktop.browser.capabilities };
const worker = await attachDesktopWebdriver(attachment);
const attached = await capture(worker.browser, artifactsRoot, 'attached');
const address = new URL(url);
const browser = await attach({
hostname: address.hostname,
port: Number(address.port),
path: '/',
sessionId: attachment.sessionId,
capabilities: attachment.capabilities,
options: { logLevel: 'silent' },
});
installDesktopWebdriverCommands(browser, url);
installDesktopWebdriverCommands(browser, url);
const managed = await capture(browser, artifactsRoot, 'managed');
process.stdout.write(JSON.stringify({ connected, attached, managed }));
} finally {
await desktop.delete();
}
}

main().catch((error) => {
process.stderr.write(error.stack ?? error.message);
process.exitCode = 1;
});
2 changes: 1 addition & 1 deletion references/agents/repository/development.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ Set up a clone with `yarn` to install dependencies, then `yarn build`.
| ------------------------------- | ----------------------------------------------------------------------------------------------------- |
| `yarn build` | `lage root-build`: runs `root-prebuild` (package `prebuild` steps), then `tsc -b` over the root graph |
| `yarn build:clean` | Cleans the root TypeScript build, then rebuilds |
| `yarn clean` | `tsc -b --clean` plus `lage clean` |
| `yarn clean` | `tsc -b --clean` plus package artifact cleanup; Lage skips the root script to prevent recursion |
| `yarn clean-lage-cache` | Clears the Lage task cache (`lage cache --clear`) |
| `yarn lage test` | Package tests after the root build and agentic contract checks |
| `yarn lage lint` | Package lint (oxlint through `fluentui-scripts lint`) |
Expand Down
2 changes: 1 addition & 1 deletion scripts/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
"bundlesize": "bundlesize --debug",
"format": "node ./src/cli.ts format",
"lint": "node ./src/cli.ts lint",
"test": "node --test src/tasks/agentInstructions.test.ts src/utils/jestSelection.test.ts"
"test": "node --test src/tasks/agentInstructions.test.ts src/tasks/clean.test.mjs src/utils/jestSelection.test.ts"
},
"dependencies": {
"@babel/core": "catalog:",
Expand Down
11 changes: 11 additions & 0 deletions scripts/src/tasks/clean.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
import assert from 'node:assert/strict';
import { readFileSync } from 'node:fs';
import { it } from 'node:test';

it('cleans TypeScript output and package artifacts without rescheduling the root script', async () => {
const { default: config } = await import(new URL('../../../lage.config.mjs', import.meta.url).href);
const root = JSON.parse(readFileSync(new URL('../../../package.json', import.meta.url), 'utf8'));
assert.equal(root.scripts.clean, 'tsc -b --clean && lage clean');
assert.deepEqual(config.pipeline[`${root.name}#clean`], { type: 'noop', cache: false });
assert.deepEqual(config.pipeline.clean, { cache: false });
});
Loading