Skip to content

Commit efc7f2e

Browse files
authored
Add MLPerf Tiny streaming wakeword model (pytorch#23133)
### Summary Add the fifth MLPerf Tiny architecture to the example model registry. The PyTorch model consumes one LFBE frame per call and retains each convolution block's history in mutable buffers, reusing earlier activations across calls. It exposes `reset()` for new streams; outputs become valid after 30 frames. This PR includes the model, its registration, and eager tests. Documentation covering all five MLPerf Tiny models will follow separately, as will ExecuTorch export, backend lowering, and runner support. The example uses randomly initialized weights. Authored with AI assistance from Codex. ### Test plan `OMP_NUM_THREADS=1 PYTHONPATH="$PWD/src" python -m pytest examples/models/test/test_streaming_wakeword.py` — 7 passed. Covers equivalence to complete 30-frame windows for random, zero, impulse, and alternating inputs; 100-frame reset/replay and independent stream state; model-factory probabilities; and rejection of multi-frame inputs. `lintrunner --revision 7970222` — passed for the changed files.
1 parent d1c0b0d commit efc7f2e

6 files changed

Lines changed: 205 additions & 0 deletions

File tree

‎examples/models/BUCK‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ fbcode_target(_kind = python_library,
1919
"//executorch/examples/models/inception_v4:ic4_model", # @manual
2020
"//executorch/examples/models/llama:llama2_model", # @manual
2121
"//executorch/examples/models/mobilebert:mobilebert_model", # @manual
22+
"//executorch/examples/models/mlperf_tiny:models", # @manual
2223
"//executorch/examples/models/mobilenet_v2:mv2_model", # @manual
2324
"//executorch/examples/models/mobilenet_v3:mv3_model", # @manual
2425
"//executorch/examples/models/resnet:resnet_model", # @manual

‎examples/models/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ class Model(str, Enum):
4444
DSCNN = "ds_cnn"
4545
MobileNetV1025 = "mobilenet_v1_025"
4646
ResNet8 = "resnet8"
47+
StreamingWakeWord = "streaming_wakeword"
4748
Sdpa = "sdpa"
4849
Qwen3 = "qwen3"
4950
SmolVLM = "smolvlm"
@@ -108,6 +109,10 @@ def __str__(self) -> str:
108109
"MobileNetV1025Model",
109110
),
110111
str(Model.ResNet8): ("mlperf_tiny.resnet8", "ResNet8Model"),
112+
str(Model.StreamingWakeWord): (
113+
"mlperf_tiny.streaming_wakeword",
114+
"StreamingWakeWordModel",
115+
),
111116
str(Model.Sdpa): ("toy_model", "SdpaModule"),
112117
str(Model.Qwen3): ("qwen3", "Qwen3Model"),
113118
str(Model.SmolVLM): ("smolvlm", "SmolVLMModel"),

‎examples/models/mlperf_tiny/BUCK‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
load("@fbcode_macros//build_defs:build_file_migration.bzl", "fbcode_target")
2+
load("@fbcode_macros//build_defs:python_library.bzl", "python_library")
3+
4+
oncall("executorch")
5+
6+
fbcode_target(
7+
_kind = python_library,
8+
name = "models",
9+
srcs = [
10+
"__init__.py",
11+
"deep_autoencoder.py",
12+
"ds_cnn.py",
13+
"mobilenet_v1_025.py",
14+
"resnet8.py",
15+
"streaming_wakeword.py",
16+
],
17+
deps = [
18+
"//caffe2:torch",
19+
"//executorch/examples/models:model_base",
20+
],
21+
)

‎examples/models/mlperf_tiny/__init__.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from .ds_cnn import DSCNNKWS, DSCNNKWSModel
88
from .mobilenet_v1_025 import MobileNetV1025, MobileNetV1025Model
99
from .resnet8 import ResNet8, ResNet8Model
10+
from .streaming_wakeword import StreamingWakeWord, StreamingWakeWordModel
1011

1112
__all__ = [
1213
"DeepAutoEncoder",
@@ -17,4 +18,6 @@
1718
"MobileNetV1025Model",
1819
"ResNet8",
1920
"ResNet8Model",
21+
"StreamingWakeWord",
22+
"StreamingWakeWordModel",
2023
]
Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
"""Incremental implementation of the MLPerf Tiny Streaming Wakeword model."""
8+
9+
from typing import cast
10+
11+
import torch
12+
13+
from executorch.examples.models.model_base import EagerModelBase
14+
from torch import nn
15+
16+
17+
class StreamingDepthwiseSeparableConv(nn.Module):
18+
history: torch.Tensor
19+
20+
def __init__(self, in_channels: int, out_channels: int, kernel_size: int):
21+
super().__init__()
22+
self.register_buffer("history", torch.zeros(1, in_channels, kernel_size - 1, 1))
23+
self.depthwise = nn.Conv2d(
24+
in_channels,
25+
in_channels,
26+
(kernel_size, 1),
27+
groups=in_channels,
28+
bias=False,
29+
)
30+
self.pointwise = nn.Conv2d(in_channels, out_channels, 1, bias=False)
31+
self.bn = nn.BatchNorm2d(out_channels, eps=1e-3, momentum=0.01)
32+
self.relu = nn.ReLU()
33+
self.dropout = nn.Dropout(0.2)
34+
35+
def forward(self, frame: torch.Tensor) -> torch.Tensor:
36+
window = torch.cat((self.history, frame), dim=2)
37+
result = self.dropout(
38+
self.relu(self.bn(self.pointwise(self.depthwise(window))))
39+
)
40+
self.history.copy_(window[:, :, 1:, :])
41+
return result
42+
43+
44+
class StreamingWakeWord(nn.Module):
45+
"""Consume one LFBE frame per call; outputs are valid after 30 frames.
46+
47+
Classes are wakeword (Marvin), silence, and other, in that order.
48+
Call reset() before an independent stream. The first 29 outputs after
49+
reset are warm-up outputs, including when the stream ends earlier.
50+
"""
51+
52+
FRAME_SHAPE = (1, 40, 1, 1)
53+
RECEPTIVE_FIELD = 30
54+
HISTORY_NAMES = tuple(f"blocks.{i}.history" for i in range(4))
55+
56+
def __init__(self, *, apply_softmax: bool = False):
57+
super().__init__()
58+
self.apply_softmax = apply_softmax
59+
self.blocks = nn.ModuleList(
60+
[
61+
StreamingDepthwiseSeparableConv(40, 128, 3),
62+
StreamingDepthwiseSeparableConv(128, 128, 5),
63+
StreamingDepthwiseSeparableConv(128, 128, 10),
64+
StreamingDepthwiseSeparableConv(128, 32, 15),
65+
]
66+
)
67+
self.classifier = nn.Linear(32, 3)
68+
69+
def forward(self, frame: torch.Tensor) -> torch.Tensor:
70+
if frame.shape != self.FRAME_SHAPE:
71+
raise ValueError(f"Expected an LFBE frame of shape {self.FRAME_SHAPE}")
72+
for block in self.blocks:
73+
frame = block(frame)
74+
logits = self.classifier(frame.flatten(1))
75+
return torch.softmax(logits, dim=-1) if self.apply_softmax else logits
76+
77+
def reset(self) -> None:
78+
for block in self.blocks:
79+
cast(torch.Tensor, block.history).zero_()
80+
81+
82+
class StreamingWakeWordModel(EagerModelBase):
83+
def get_eager_model(self) -> nn.Module:
84+
return StreamingWakeWord(apply_softmax=True).eval()
85+
86+
def get_example_inputs(self):
87+
return (torch.rand(StreamingWakeWord.FRAME_SHAPE) * 2 - 1,)
Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
# Copyright (c) Meta Platforms, Inc. and affiliates.
2+
# All rights reserved.
3+
#
4+
# This source code is licensed under the BSD-style license found in the
5+
# LICENSE file in the root directory of this source tree.
6+
7+
import copy
8+
9+
import pytest
10+
import torch
11+
from executorch.examples.models import MODEL_NAME_TO_MODEL
12+
from executorch.examples.models.mlperf_tiny.streaming_wakeword import StreamingWakeWord
13+
from executorch.examples.models.model_factory import EagerModelFactory
14+
from torch import nn
15+
16+
17+
@pytest.fixture
18+
def model():
19+
torch.manual_seed(42)
20+
model = StreamingWakeWord().eval()
21+
with torch.no_grad():
22+
for block in model.blocks:
23+
block.bn.bias.uniform_(-0.1, 0.1)
24+
block.bn.running_mean.uniform_(-0.2, 0.2)
25+
block.bn.running_var.uniform_(0.5, 1.5)
26+
return model
27+
28+
29+
@pytest.mark.parametrize("signal", ["random", "zero", "impulse", "alternating"])
30+
@torch.no_grad()
31+
def test_matches_full_windows(model, signal):
32+
layers = []
33+
for block in model.blocks:
34+
layers.extend(
35+
copy.deepcopy(layer)
36+
for layer in (block.depthwise, block.pointwise, block.bn, block.relu)
37+
)
38+
reference = nn.Sequential(*layers, nn.Flatten(1), copy.deepcopy(model.classifier))
39+
frames = torch.randn(1, 40, 70, 1)
40+
if signal == "zero":
41+
frames.zero_()
42+
elif signal == "impulse":
43+
frames.zero_()
44+
frames[:, :, 30, :] = 1
45+
elif signal == "alternating":
46+
frames[:, :, ::2, :] = 1
47+
frames[:, :, 1::2, :] = -1
48+
49+
for index in range(frames.shape[2]):
50+
actual = model(frames[:, :, index : index + 1, :])
51+
if index >= model.RECEPTIVE_FIELD - 1:
52+
window = frames[:, :, index - model.RECEPTIVE_FIELD + 1 : index + 1, :]
53+
torch.testing.assert_close(actual, reference(window), atol=1e-6, rtol=1e-5)
54+
55+
56+
@torch.no_grad()
57+
def test_reset_replay_and_independent_streams(model):
58+
inputs = [torch.randn(model.FRAME_SHAPE) for _ in range(100)]
59+
independent = StreamingWakeWord().eval()
60+
independent.load_state_dict(model.state_dict())
61+
expected = [model(frame).clone() for frame in inputs]
62+
63+
for frame, result in zip(inputs[:35], expected[:35]):
64+
torch.testing.assert_close(independent(frame), result)
65+
66+
model.reset()
67+
assert all(torch.count_nonzero(block.history) == 0 for block in model.blocks)
68+
for frame, result in zip(inputs[35:], expected[35:]):
69+
torch.testing.assert_close(independent(frame), result)
70+
71+
for frame, result in zip(inputs, expected):
72+
torch.testing.assert_close(model(frame), result)
73+
74+
75+
@torch.no_grad()
76+
def test_factory_returns_probabilities():
77+
example, inputs, _, _ = EagerModelFactory.create_model(
78+
*MODEL_NAME_TO_MODEL["streaming_wakeword"]
79+
)
80+
assert not example.training
81+
output = example(*inputs)
82+
assert output.shape == (1, 3)
83+
torch.testing.assert_close(output.sum(dim=-1), torch.ones(1))
84+
85+
86+
def test_rejects_multi_frame_input(model):
87+
with pytest.raises(ValueError, match="LFBE frame"):
88+
model(torch.zeros(1, 40, 2, 1))

0 commit comments

Comments
 (0)