From 7702dc189d08c02cb33db9eabfbcdc546b0f2ddc Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:25:33 +0200 Subject: [PATCH 01/15] process_management: add PROCESS_INFO_USES_MICROPHONE flag Apps declare the microphone permission with `capabilities: ["microphone"]` in their manifest. Record that declaration in the binary header (bit 11) so the firmware can tell "not declared" apart from "denied" when it enforces the upcoming per-app permission system, and surface it through PebbleProcessMd and AppInstallEntry. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012ekdUsEpAe9dNq5wyLYhGc Signed-off-by: Joshua Jun --- sdk/tools/schemas/attributes.json | 2 +- .../process_management/app_install_manager.c | 6 ++++++ .../process_management/app_install_manager.h | 5 +++++ .../process_management/pebble_process_info.h | 2 ++ src/fw/process_management/pebble_process_md.c | 5 +++++ src/fw/process_management/pebble_process_md.h | 5 +++++ tests/fw/services/test_app_install_manager.c | 19 ++++++++++++++++++- tools/generate_appinfo.py | 4 ++++ 8 files changed, 46 insertions(+), 2 deletions(-) diff --git a/sdk/tools/schemas/attributes.json b/sdk/tools/schemas/attributes.json index a48c999596..c38dc4f56b 100644 --- a/sdk/tools/schemas/attributes.json +++ b/sdk/tools/schemas/attributes.json @@ -11,7 +11,7 @@ }, "capabilities": { "type": "array", - "items": { "enum": ["location", "configurable", "health"] }, + "items": { "enum": ["location", "configurable", "health", "microphone"] }, "uniqueItems": true }, "messageKeys": { diff --git a/src/fw/process_management/app_install_manager.c b/src/fw/process_management/app_install_manager.c index c5e7244c9e..d744cfcdbe 100644 --- a/src/fw/process_management/app_install_manager.c +++ b/src/fw/process_management/app_install_manager.c @@ -185,6 +185,10 @@ bool app_install_entry_has_worker(const AppInstallEntry *entry) { return (entry->has_worker); } +bool app_install_entry_uses_microphone(const AppInstallEntry *entry) { + return (entry->uses_microphone); +} + bool app_install_entry_is_hidden(const AppInstallEntry *entry) { switch (entry->visibility) { case ProcessVisibilityHidden: @@ -644,6 +648,7 @@ static bool prv_app_install_entry_from_app_db_entry(AppInstallId id, AppDBEntry // applications registered with the manager are applications, not workers. .process_type = process_metadata_flags_process_type(db_entry->info_flags, PebbleTask_App), .has_worker = process_metadata_flags_has_worker(db_entry->info_flags), + .uses_microphone = process_metadata_flags_uses_microphone(db_entry->info_flags), .icon_resource_id = db_entry->icon_resource_id, .uuid = db_entry->uuid, .color = prv_valid_color_from_uuid(db_entry->app_face_bg_color, (Uuid *)&db_entry->uuid), @@ -674,6 +679,7 @@ static bool prv_app_install_entry_from_resource_registry_entry(const AppRegistry // applications registered with the manager are applications, not workers. .process_type = process_metadata_flags_process_type(app_header->flags, PebbleTask_App), .has_worker = process_metadata_flags_has_worker(app_header->flags), + .uses_microphone = process_metadata_flags_uses_microphone(app_header->flags), .icon_resource_id = reg_entry->icon_resource_id, .uuid = reg_entry->uuid, .color = prv_valid_color_from_uuid(reg_entry->color, (Uuid *)®_entry->uuid), diff --git a/src/fw/process_management/app_install_manager.h b/src/fw/process_management/app_install_manager.h index 9a9c562bd7..31ce8a9079 100644 --- a/src/fw/process_management/app_install_manager.h +++ b/src/fw/process_management/app_install_manager.h @@ -82,6 +82,7 @@ typedef struct { ProcessVisibility visibility; ProcessType process_type; // WATCHFACE/APP bool has_worker; + bool uses_microphone; Uuid uuid; GColor color; char name[APP_NAME_SIZE_BYTES]; @@ -185,6 +186,10 @@ bool app_install_entry_is_watchface(const AppInstallEntry *entry); //! @param entry AppInstallEntry to check the parameters of bool app_install_entry_has_worker(const AppInstallEntry *entry); +//! Returns true if the app associated with the provided entry declares the microphone permission +//! @param entry AppInstallEntry to check the parameters of +bool app_install_entry_uses_microphone(const AppInstallEntry *entry); + //! Returns true if the app associated with the provided entry should be hidden in menus //! @param entry AppInstallEntry to check the parameters of bool app_install_entry_is_hidden(const AppInstallEntry *entry); diff --git a/src/fw/process_management/pebble_process_info.h b/src/fw/process_management/pebble_process_info.h index 63e1f2f5e8..93088d03f8 100644 --- a/src/fw/process_management/pebble_process_info.h +++ b/src/fw/process_management/pebble_process_info.h @@ -37,6 +37,8 @@ typedef enum { //! True, if process uses Moddable XS APIs PROCESS_INFO_MODDABLE_APP = 1 << 10, + //! Process declares the microphone permission (bits 12-31 are free) + PROCESS_INFO_USES_MICROPHONE = 1 << 11, //! SDK older than 4.2 doesn't store any value PROCESS_INFO_PLATFORM_UNKNOWN = 0x0 << 6, PROCESS_INFO_PLATFORM_APLITE = 0x1 << 6, diff --git a/src/fw/process_management/pebble_process_md.c b/src/fw/process_management/pebble_process_md.c index 800ffa0952..66a0b9bb42 100644 --- a/src/fw/process_management/pebble_process_md.c +++ b/src/fw/process_management/pebble_process_md.c @@ -95,6 +95,7 @@ static void prv_init_from_info_common(PebbleProcessMd *common, const PebbleProce common->has_worker = process_metadata_flags_has_worker(info->flags); common->is_moddable_app = process_metadata_flags_moddable_app(info->flags); common->is_rocky_app = process_metadata_flags_rocky_app(info->flags); + common->uses_microphone = process_metadata_flags_uses_microphone(info->flags); common->stored_sdk_platform = process_metadata_flags_stored_sdk_platform(info->flags); common->is_unprivileged = true; // We don't know the load address of the process until the process is @@ -192,6 +193,10 @@ bool process_metadata_flags_rocky_app(PebbleProcessInfoFlags flags) { return ((flags & PROCESS_INFO_ROCKY_APP) != 0); } +bool process_metadata_flags_uses_microphone(PebbleProcessInfoFlags flags) { + return ((flags & PROCESS_INFO_USES_MICROPHONE) != 0); +} + uint16_t process_metadata_flags_stored_sdk_platform(PebbleProcessInfoFlags flags) { return (flags & PROCESS_INFO_PLATFORM_MASK); } diff --git a/src/fw/process_management/pebble_process_md.h b/src/fw/process_management/pebble_process_md.h index 2fd69df7ff..023a402dcb 100644 --- a/src/fw/process_management/pebble_process_md.h +++ b/src/fw/process_management/pebble_process_md.h @@ -84,6 +84,9 @@ typedef struct PebbleProcessMd { //! Deprecated: Process was built as a RockyJS app (no longer supported) bool is_rocky_app; + //! Process declares the microphone permission in its manifest + bool uses_microphone; + //! Bits of the sdk_platform as they were stored in the binary, or 0 if undefined uint16_t stored_sdk_platform; } PebbleProcessMd; @@ -187,6 +190,8 @@ bool process_metadata_flags_moddable_app(PebbleProcessInfoFlags flags); bool process_metadata_flags_rocky_app(PebbleProcessInfoFlags flags); +bool process_metadata_flags_uses_microphone(PebbleProcessInfoFlags flags); + uint16_t process_metadata_flags_stored_sdk_platform(PebbleProcessInfoFlags flags); ProcessAppSDKType process_metadata_get_app_sdk_type(const PebbleProcessMd *md); diff --git a/tests/fw/services/test_app_install_manager.c b/tests/fw/services/test_app_install_manager.c index 3a6ab80f38..0d4631850f 100644 --- a/tests/fw/services/test_app_install_manager.c +++ b/tests/fw/services/test_app_install_manager.c @@ -171,6 +171,15 @@ bool app_install_has_worker(AppInstallId id) { return app_install_entry_has_worker(&entry); } +bool app_install_uses_microphone(AppInstallId id) { + AppInstallEntry entry; + bool exists = app_install_get_entry_for_install_id(id, &entry); + if (!exists) { + return false; + } + return app_install_entry_uses_microphone(&entry); +} + bool app_install_is_hidden(AppInstallId id) { AppInstallEntry entry; bool exists = app_install_get_entry_for_install_id(id, &entry); @@ -199,7 +208,7 @@ static const AppDBEntry bg_counter = { .uuid = {0x1e, 0xb1, 0xd3, 0x9b, 0x56, 0x98, 0x48, 0x44, 0xb3, 0x94, 0x1f, 0x87, 0xb6, 0xbe, 0xae, 0x67}, - .info_flags = PROCESS_INFO_HAS_WORKER | PROCESS_INFO_STANDARD_APP, + .info_flags = PROCESS_INFO_HAS_WORKER | PROCESS_INFO_USES_MICROPHONE | PROCESS_INFO_STANDARD_APP, .app_version = { .major = 1, @@ -391,6 +400,14 @@ void test_app_install_manager__has_worker(void) { cl_assert_equal_b(false, app_install_has_worker(CRAZY_ID)); } +void test_app_install_manager__uses_microphone(void) { + cl_assert_equal_b(false, app_install_uses_microphone(tictoc_id)); + cl_assert_equal_b(false, app_install_uses_microphone(music_id)); + cl_assert_equal_b(true, app_install_uses_microphone(bg_counter_id)); + cl_assert_equal_b(false, app_install_uses_microphone(menu_layer_id)); + cl_assert_equal_b(false, app_install_uses_microphone(CRAZY_ID)); +} + void test_app_install_manager__is_hidden(void) { cl_assert_equal_b(false, app_install_is_hidden(tictoc_id)); cl_assert_equal_b(false, app_install_is_hidden(music_id)); diff --git a/tools/generate_appinfo.py b/tools/generate_appinfo.py index aeb6f58b4f..dae4cac04a 100644 --- a/tools/generate_appinfo.py +++ b/tools/generate_appinfo.py @@ -115,6 +115,10 @@ def generate_appinfo_c(app_info, output_filename, platform_name=None): flags.append("PROCESS_INFO_VISIBILITY_HIDDEN") if is_moddable: flags.append("PROCESS_INFO_MODDABLE_APP") + if "microphone" in app_info.get("capabilities", []): + if is_watchface: + raise ValueError("Watchfaces cannot use the microphone; remove the 'microphone' capability") + flags.append("PROCESS_INFO_USES_MICROPHONE") if platform_name: flags.append(f"PROCESS_INFO_PLATFORM_{platform_name.upper()}") From 69d3b8fe2b017091081ae1268517ea3069d12da8 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:28:54 +0200 Subject: [PATCH 02/15] blob_db: add AppPermissions database Adds BlobDBIdAppPermissions (0x0D), a settings-file backed store of per-app permission grants keyed by app UUID. The phone is the source of truth and pushes a small versioned record with a granted mask and a declared mask. Records are removed when the app is uninstalled, but kept across upgrades so grants persist. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012ekdUsEpAe9dNq5wyLYhGc Signed-off-by: Joshua Jun --- .../app_permissions/app_permissions_types.h | 53 +++++ include/pbl/services/blob_db/api.h | 1 + .../pbl/services/blob_db/app_permissions_db.h | 51 +++++ .../process_management/app_install_manager.c | 2 + src/fw/services/blob_db/CMakeLists.txt | 1 + src/fw/services/blob_db/api.c | 34 ++- src/fw/services/blob_db/app_permissions_db.c | 198 ++++++++++++++++++ src/fw/shell/prf/stubs.c | 5 + tests/fw/services/blob_db/CMakeLists.txt | 18 ++ .../blob_db/test_app_permissions_db.c | 195 +++++++++++++++++ .../services/blob_db/test_blob_db2_endpoint.c | 1 + .../services/blob_db/test_blob_db_endpoint.c | 1 + tests/fw/services/test_app_install_manager.c | 1 + tests/fw/services/test_app_menu_data_source.c | 1 + tests/stubs/stubs_app_permissions_db.h | 55 +++++ 15 files changed, 606 insertions(+), 11 deletions(-) create mode 100644 include/pbl/services/app_permissions/app_permissions_types.h create mode 100644 include/pbl/services/blob_db/app_permissions_db.h create mode 100644 src/fw/services/blob_db/app_permissions_db.c create mode 100644 tests/fw/services/blob_db/test_app_permissions_db.c create mode 100644 tests/stubs/stubs_app_permissions_db.h diff --git a/include/pbl/services/app_permissions/app_permissions_types.h b/include/pbl/services/app_permissions/app_permissions_types.h new file mode 100644 index 0000000000..075011d93a --- /dev/null +++ b/include/pbl/services/app_permissions/app_permissions_types.h @@ -0,0 +1,53 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "pbl/util/attributes.h" + +#include + +//! @addtogroup Foundation +//! @{ +//! @addtogroup Permissions +//! @{ + +//! Permissions an app can declare in its manifest and the user can grant or deny. +typedef enum AppPermission { + //! Live microphone capture (`capabilities: ["microphone"]`) + AppPermission_Microphone = 0, + AppPermissionCount, +} AppPermission; + +//! State of a permission for one app. +typedef enum AppPermissionState { + //! The app does not declare the permission in its manifest. + AppPermissionStateNotDeclared = 0, + //! The permission is declared but has not been granted by the user. + AppPermissionStateDenied = 1, + //! The permission is granted. + AppPermissionStateGranted = 2, +} AppPermissionState; + +//! @} // end addtogroup Permissions +//! @} // end addtogroup Foundation + +//! @internal +typedef uint32_t AppPermissionMask; + +//! @internal +#define APP_PERMISSION_BIT(permission) ((AppPermissionMask)1u << (permission)) + +//! @internal +#define APP_PERMISSIONS_DB_ENTRY_VERSION (1) + +//! @internal +//! Serialized grant record pushed by the phone (BlobDBIdAppPermissions, keyed by app Uuid). +typedef struct PACKED AppPermissionsDBEntry { + uint8_t version; //!< APP_PERMISSIONS_DB_ENTRY_VERSION + uint8_t reserved[3]; + AppPermissionMask granted_mask; //!< Permissions the user granted + AppPermissionMask declared_mask; //!< Permissions the app declared (informational) +} AppPermissionsDBEntry; + +_Static_assert(sizeof(AppPermissionsDBEntry) == 12, "AppPermissionsDBEntry layout changed"); diff --git a/include/pbl/services/blob_db/api.h b/include/pbl/services/blob_db/api.h index 0cb7cc3557..32ff629f76 100644 --- a/include/pbl/services/blob_db/api.h +++ b/include/pbl/services/blob_db/api.h @@ -36,6 +36,7 @@ typedef enum PACKED { BlobDBIdHealth = 0x0A, BlobDBIdAppGlance = 0x0B, BlobDBIdSettings = 0x0C, + BlobDBIdAppPermissions = 0x0D, NumBlobDBs, } BlobDBId; _Static_assert(sizeof(BlobDBId) == 1, "BlobDBId is larger than 1 byte"); diff --git a/include/pbl/services/blob_db/app_permissions_db.h b/include/pbl/services/blob_db/app_permissions_db.h new file mode 100644 index 0000000000..09bac1bd58 --- /dev/null +++ b/include/pbl/services/blob_db/app_permissions_db.h @@ -0,0 +1,51 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "pbl/services/app_permissions/app_permissions_types.h" +#include "pbl/util/uuid.h" +#include "system/status_codes.h" + +#include +#include + +//! Per-app permission grants, pushed by the phone and keyed by app Uuid. + +//! Reads the grant record for an app. +//! @return S_SUCCESS, E_DOES_NOT_EXIST if there is no record, or another error +status_t app_permissions_db_get(const Uuid *uuid, AppPermissionsDBEntry *entry_out); + +//! Writes a grant record for an app. Goes through the generic BlobDB insert path so the same +//! events fire as for a phone-originated insert. +status_t app_permissions_db_set(const Uuid *uuid, const AppPermissionsDBEntry *entry); + +//! Deletes the grant record for an app, if any. Goes through the generic BlobDB delete path. +status_t app_permissions_db_delete_for_uuid(const Uuid *uuid); + +//! Return false to stop the iteration. +typedef bool (*AppPermissionsDBEachCallback)(const Uuid *uuid, const AppPermissionsDBEntry *entry, + void *context); + +//! Iterates over every grant record. +status_t app_permissions_db_each(AppPermissionsDBEachCallback cb, void *context); + +/////////////////////////////////////////// +// BlobDB Boilerplate (see blob_db/api.h) +/////////////////////////////////////////// + +void app_permissions_db_init(void); + +status_t app_permissions_db_insert(const uint8_t *key, int key_len, const uint8_t *val, + int val_len); + +int app_permissions_db_get_len(const uint8_t *key, int key_len); + +status_t app_permissions_db_read(const uint8_t *key, int key_len, uint8_t *val_out, + int val_out_len); + +status_t app_permissions_db_delete(const uint8_t *key, int key_len); + +status_t app_permissions_db_flush(void); + +status_t app_permissions_db_compact(void); diff --git a/src/fw/process_management/app_install_manager.c b/src/fw/process_management/app_install_manager.c index d744cfcdbe..6cbb03a8fc 100644 --- a/src/fw/process_management/app_install_manager.c +++ b/src/fw/process_management/app_install_manager.c @@ -21,6 +21,7 @@ #include "pbl/services/i18n/i18n.h" #include "pbl/services/app_cache.h" #include "pbl/services/blob_db/app_db.h" +#include "pbl/services/blob_db/app_permissions_db.h" #include "pbl/services/blob_db/pin_db.h" #include "pbl/services/persist.h" #include "pbl/services/process_management/app_storage.h" @@ -437,6 +438,7 @@ static void app_install_launcher_task_callback(void *context) { // app, not during an AppDB clear. if (!app_upgrade) { persist_service_delete_file(s_install_callback_data.uuid); + app_permissions_db_delete_for_uuid(s_install_callback_data.uuid); #if !defined(CONFIG_RECOVERY_FW) comm_session_app_session_capabilities_evict(s_install_callback_data.uuid); #endif diff --git a/src/fw/services/blob_db/CMakeLists.txt b/src/fw/services/blob_db/CMakeLists.txt index 9f5ff20526..75a8ab98cd 100644 --- a/src/fw/services/blob_db/CMakeLists.txt +++ b/src/fw/services/blob_db/CMakeLists.txt @@ -5,6 +5,7 @@ set(sources api.c app_db.c app_glance_db.c + app_permissions_db.c contacts_db.c endpoint.c endpoint2.c diff --git a/src/fw/services/blob_db/api.c b/src/fw/services/blob_db/api.c index b6b28920b7..c40dcbebcb 100644 --- a/src/fw/services/blob_db/api.c +++ b/src/fw/services/blob_db/api.c @@ -8,6 +8,7 @@ #include "pbl/services/blob_db/app_db.h" #include "pbl/services/blob_db/app_glance_db.h" +#include "pbl/services/blob_db/app_permissions_db.h" #include "pbl/services/blob_db/contacts_db.h" #include "pbl/services/blob_db/health_db.h" #include "pbl/services/blob_db/ios_notif_pref_db.h" @@ -171,17 +172,28 @@ static const BlobDB s_blob_dbs[NumBlobDBs] = { .compact = app_glance_db_compact, .name = "app_glance_db", }, - [BlobDBIdSettings] = { - .init = settings_blob_db_init, - .insert = settings_blob_db_insert, - .get_len = settings_blob_db_get_len, - .read = settings_blob_db_read, - .del = settings_blob_db_delete, - .flush = settings_blob_db_flush, - .is_dirty = settings_blob_db_is_dirty, - .get_dirty_list = settings_blob_db_get_dirty_list, - .mark_synced = settings_blob_db_mark_synced, - .name = "settings_blob_db", + [BlobDBIdSettings] = + { + .init = settings_blob_db_init, + .insert = settings_blob_db_insert, + .get_len = settings_blob_db_get_len, + .read = settings_blob_db_read, + .del = settings_blob_db_delete, + .flush = settings_blob_db_flush, + .is_dirty = settings_blob_db_is_dirty, + .get_dirty_list = settings_blob_db_get_dirty_list, + .mark_synced = settings_blob_db_mark_synced, + .name = "settings_blob_db", + }, + [BlobDBIdAppPermissions] = { + .init = app_permissions_db_init, + .insert = app_permissions_db_insert, + .get_len = app_permissions_db_get_len, + .read = app_permissions_db_read, + .del = app_permissions_db_delete, + .flush = app_permissions_db_flush, + .compact = app_permissions_db_compact, + .name = "app_permissions_db", }, }; diff --git a/src/fw/services/blob_db/app_permissions_db.c b/src/fw/services/blob_db/app_permissions_db.c new file mode 100644 index 0000000000..ecbc373bd4 --- /dev/null +++ b/src/fw/services/blob_db/app_permissions_db.c @@ -0,0 +1,198 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "pbl/services/blob_db/app_permissions_db.h" + +#include "pbl/kernel/mutex.h" +#include "pbl/services/blob_db/api.h" +#include "pbl/services/filesystem/pfs.h" +#include "pbl/services/settings/settings_file.h" +#include "util/units.h" +#include + +#include + +PBL_LOG_MODULE_DECLARE(service_blob_db, CONFIG_SERVICE_BLOB_DB_LOG_LEVEL); + +#define SETTINGS_FILE_NAME "app_permissions" +#define SETTINGS_FILE_SIZE KiBYTES(8) + +static struct { + SettingsFile settings_file; + struct pbl_mutex mutex; +} s_app_permissions_db; + +static status_t prv_lock_mutex_and_open_file(void) { + pbl_mutex_lock(&s_app_permissions_db.mutex, PBL_FOREVER); + status_t rv = settings_file_open_growable(&s_app_permissions_db.settings_file, SETTINGS_FILE_NAME, + SETTINGS_FILE_SIZE, KiBYTES(2)); + if (rv != S_SUCCESS) { + pbl_mutex_unlock(&s_app_permissions_db.mutex); + } + return rv; +} + +static void prv_close_file_and_unlock_mutex(void) { + settings_file_close(&s_app_permissions_db.settings_file); + pbl_mutex_unlock(&s_app_permissions_db.mutex); +} + +static bool prv_is_key_valid(int key_len) { + return (key_len == UUID_SIZE); +} + +static bool prv_is_val_valid(const uint8_t *val, int val_len) { + if (!val || (val_len < (int)sizeof(AppPermissionsDBEntry))) { + return false; + } + return (((const AppPermissionsDBEntry *)val)->version == APP_PERMISSIONS_DB_ENTRY_VERSION); +} + +// Convenience API +//////////////////////////////////////////////////////////////////////////////// + +status_t app_permissions_db_get(const Uuid *uuid, AppPermissionsDBEntry *entry_out) { + if (!uuid || !entry_out) { + return E_INVALID_ARGUMENT; + } + status_t rv = prv_lock_mutex_and_open_file(); + if (rv != S_SUCCESS) { + return rv; + } + const int len = settings_file_get_len(&s_app_permissions_db.settings_file, uuid, sizeof(*uuid)); + if (len < (int)sizeof(*entry_out)) { + rv = E_DOES_NOT_EXIST; + } else { + rv = settings_file_get(&s_app_permissions_db.settings_file, uuid, sizeof(*uuid), entry_out, + sizeof(*entry_out)); + } + prv_close_file_and_unlock_mutex(); + return rv; +} + +status_t app_permissions_db_set(const Uuid *uuid, const AppPermissionsDBEntry *entry) { + if (!uuid || !entry) { + return E_INVALID_ARGUMENT; + } + return blob_db_insert(BlobDBIdAppPermissions, (const uint8_t *)uuid, sizeof(*uuid), + (const uint8_t *)entry, sizeof(*entry)); +} + +status_t app_permissions_db_delete_for_uuid(const Uuid *uuid) { + if (!uuid) { + return E_INVALID_ARGUMENT; + } + return blob_db_delete(BlobDBIdAppPermissions, (const uint8_t *)uuid, sizeof(*uuid)); +} + +typedef struct { + AppPermissionsDBEachCallback cb; + void *context; +} EachContext; + +static bool prv_each_record(SettingsFile *file, SettingsRecordInfo *info, void *context) { + EachContext *each = context; + if ((info->key_len != UUID_SIZE) || (info->val_len < (int)sizeof(AppPermissionsDBEntry))) { + return true; + } + Uuid uuid; + AppPermissionsDBEntry entry; + info->get_key(file, &uuid, sizeof(uuid)); + info->get_val(file, &entry, sizeof(entry)); + return each->cb(&uuid, &entry, each->context); +} + +status_t app_permissions_db_each(AppPermissionsDBEachCallback cb, void *context) { + if (!cb) { + return E_INVALID_ARGUMENT; + } + status_t rv = prv_lock_mutex_and_open_file(); + if (rv != S_SUCCESS) { + return rv; + } + EachContext each = {.cb = cb, .context = context}; + rv = settings_file_each(&s_app_permissions_db.settings_file, prv_each_record, &each); + prv_close_file_and_unlock_mutex(); + return rv; +} + +// BlobDB APIs +//////////////////////////////////////////////////////////////////////////////// + +void app_permissions_db_init(void) { + pbl_mutex_init(&s_app_permissions_db.mutex); +} + +status_t app_permissions_db_insert(const uint8_t *key, int key_len, const uint8_t *val, + int val_len) { + if (!prv_is_key_valid(key_len)) { + PBL_LOG_ERR("Error inserting app permission: invalid key"); + return E_INVALID_ARGUMENT; + } + if (!prv_is_val_valid(val, val_len)) { + PBL_LOG_ERR("Error inserting app permission: invalid value"); + return E_INVALID_ARGUMENT; + } + + status_t rv = prv_lock_mutex_and_open_file(); + if (rv == S_SUCCESS) { + rv = settings_file_set(&s_app_permissions_db.settings_file, key, key_len, val, + sizeof(AppPermissionsDBEntry)); + prv_close_file_and_unlock_mutex(); + } + return rv; +} + +int app_permissions_db_get_len(const uint8_t *key, int key_len) { + if (!prv_is_key_valid(key_len)) { + return 0; + } + int len = 0; + if (prv_lock_mutex_and_open_file() == S_SUCCESS) { + len = settings_file_get_len(&s_app_permissions_db.settings_file, key, key_len); + prv_close_file_and_unlock_mutex(); + } + return len; +} + +status_t app_permissions_db_read(const uint8_t *key, int key_len, uint8_t *val_out, + int val_out_len) { + if (!prv_is_key_valid(key_len) || !val_out) { + return E_INVALID_ARGUMENT; + } + status_t rv = prv_lock_mutex_and_open_file(); + if (rv == S_SUCCESS) { + rv = settings_file_get(&s_app_permissions_db.settings_file, key, key_len, val_out, val_out_len); + prv_close_file_and_unlock_mutex(); + } + return rv; +} + +status_t app_permissions_db_delete(const uint8_t *key, int key_len) { + if (!prv_is_key_valid(key_len)) { + return E_INVALID_ARGUMENT; + } + status_t rv = prv_lock_mutex_and_open_file(); + if (rv == S_SUCCESS) { + rv = settings_file_delete(&s_app_permissions_db.settings_file, key, key_len); + prv_close_file_and_unlock_mutex(); + } + return rv; +} + +status_t app_permissions_db_flush(void) { + pbl_mutex_lock(&s_app_permissions_db.mutex, PBL_FOREVER); + status_t rv = pfs_remove(SETTINGS_FILE_NAME); + pbl_mutex_unlock(&s_app_permissions_db.mutex); + return rv; +} + +status_t app_permissions_db_compact(void) { + status_t rv = prv_lock_mutex_and_open_file(); + if (rv != S_SUCCESS) { + return rv; + } + rv = settings_file_compact(&s_app_permissions_db.settings_file); + prv_close_file_and_unlock_mutex(); + return rv; +} diff --git a/src/fw/shell/prf/stubs.c b/src/fw/shell/prf/stubs.c index 8e2d66141b..dd66c54fe8 100644 --- a/src/fw/shell/prf/stubs.c +++ b/src/fw/shell/prf/stubs.c @@ -19,6 +19,7 @@ #include "pbl/services/light.h" #include "pbl/services/notifications/do_not_disturb.h" #include "pbl/services/notifications/alerts_private.h" +#include "pbl/services/blob_db/app_permissions_db.h" #include "pbl/services/persist.h" #include "shell/prefs.h" #include "shell/system_theme.h" @@ -83,6 +84,10 @@ status_t persist_service_delete_file(const Uuid *uuid) { return E_INVALID_OPERATION; } +status_t app_permissions_db_delete_for_uuid(const Uuid *uuid) { + return E_INVALID_OPERATION; +} + void wakeup_enable(bool enable) { } diff --git a/tests/fw/services/blob_db/CMakeLists.txt b/tests/fw/services/blob_db/CMakeLists.txt index f32539ca12..4facb2d7ed 100644 --- a/tests/fw/services/blob_db/CMakeLists.txt +++ b/tests/fw/services/blob_db/CMakeLists.txt @@ -131,6 +131,24 @@ pbl_clar_test(test_watch_app_prefs_db OVERRIDES dummy_board ) +pbl_clar_test(test_app_permissions_db + SOURCES + src/fw/util/crc8.c + src/fw/util/legacy_checksum.c + tests/fakes/fake_spi_flash.c + src/fw/util/rand/rand.c + third_party/tinymt/TinyMT/tinymt/tinymt32.c + tests/fakes/fake_rtc.c + src/fw/flash_region/flash_region.c + src/fw/flash_region/filesystem_regions.c + src/fw/services/settings/settings_file.c + src/fw/services/settings/settings_raw_iter.c + src/fw/services/filesystem/flash_translation.c + src/fw/services/filesystem/pfs.c + src/fw/services/blob_db/app_permissions_db.c + OVERRIDES dummy_board +) + pbl_clar_test(test_reminder_db SOURCES src/fw/util/crc8.c diff --git a/tests/fw/services/blob_db/test_app_permissions_db.c b/tests/fw/services/blob_db/test_app_permissions_db.c new file mode 100644 index 0000000000..f91eed07e6 --- /dev/null +++ b/tests/fw/services/blob_db/test_app_permissions_db.c @@ -0,0 +1,195 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "clar.h" + +#include "pbl/services/blob_db/api.h" +#include "pbl/services/blob_db/app_permissions_db.h" +#include "pbl/services/filesystem/pfs.h" +#include "pbl/util/uuid.h" + +#include + +// Fakes +//////////////////////////////////////////////////////////////// +#include "fake_spi_flash.h" +#include "fake_system_task.h" +#include "fake_kernel_services_notifications.h" + +// Stubs +//////////////////////////////////////////////////////////////// +#include "stubs_analytics.h" +#include "stubs_hexdump.h" +#include "stubs_layout_layer.h" +#include "stubs_logging.h" +#include "stubs_mutex.h" +#include "stubs_passert.h" +#include "stubs_pbl_malloc.h" +#include "stubs_prompt.h" +#include "stubs_rand_ptr.h" +#include "stubs_sleep.h" +#include "stubs_task_watchdog.h" + +// The convenience setters route through the generic BlobDB API so events fire; here we short +// circuit straight into the implementation and record the calls. +static int s_num_inserts; +static int s_num_deletes; + +status_t blob_db_insert(BlobDBId db_id, const uint8_t *key, int key_len, const uint8_t *val, + int val_len) { + cl_assert_equal_i(db_id, BlobDBIdAppPermissions); + s_num_inserts++; + return app_permissions_db_insert(key, key_len, val, val_len); +} + +status_t blob_db_delete(BlobDBId db_id, const uint8_t *key, int key_len) { + cl_assert_equal_i(db_id, BlobDBIdAppPermissions); + s_num_deletes++; + return app_permissions_db_delete(key, key_len); +} + +static const Uuid s_uuid_a = {0x1e, 0xb1, 0xd3, 0x9b, 0x56, 0x98, 0x48, 0x44, + 0xb3, 0x94, 0x1f, 0x87, 0xb6, 0xbe, 0xae, 0x67}; +static const Uuid s_uuid_b = {0xb8, 0x26, 0x2e, 0x08, 0x57, 0xe9, 0x4e, 0x58, + 0x88, 0x02, 0x45, 0xfd, 0xfe, 0xe0, 0xac, 0x77}; + +static const AppPermissionsDBEntry s_granted_mic = { + .version = APP_PERMISSIONS_DB_ENTRY_VERSION, + .granted_mask = APP_PERMISSION_BIT(AppPermission_Microphone), + .declared_mask = APP_PERMISSION_BIT(AppPermission_Microphone), +}; + +static const AppPermissionsDBEntry s_denied_mic = { + .version = APP_PERMISSIONS_DB_ENTRY_VERSION, + .granted_mask = 0, + .declared_mask = APP_PERMISSION_BIT(AppPermission_Microphone), +}; + +// Setup +//////////////////////////////////////////////////////////////// + +void test_app_permissions_db__initialize(void) { + fake_spi_flash_init(0, 0x1000000); + pfs_init(false); + app_permissions_db_init(); + s_num_inserts = 0; + s_num_deletes = 0; +} + +void test_app_permissions_db__cleanup(void) { +} + +// Tests +//////////////////////////////////////////////////////////////// + +void test_app_permissions_db__insert_read_roundtrip(void) { + cl_assert_equal_i( + S_SUCCESS, app_permissions_db_insert((const uint8_t *)&s_uuid_a, UUID_SIZE, + (const uint8_t *)&s_granted_mic, sizeof(s_granted_mic))); + cl_assert_equal_i(sizeof(AppPermissionsDBEntry), + app_permissions_db_get_len((const uint8_t *)&s_uuid_a, UUID_SIZE)); + + AppPermissionsDBEntry entry = {}; + cl_assert_equal_i(S_SUCCESS, app_permissions_db_read((const uint8_t *)&s_uuid_a, UUID_SIZE, + (uint8_t *)&entry, sizeof(entry))); + cl_assert_equal_m((void *)&entry, (void *)&s_granted_mic, sizeof(entry)); + + // Overwrite with a denial + cl_assert_equal_i( + S_SUCCESS, app_permissions_db_insert((const uint8_t *)&s_uuid_a, UUID_SIZE, + (const uint8_t *)&s_denied_mic, sizeof(s_denied_mic))); + cl_assert_equal_i(S_SUCCESS, app_permissions_db_get(&s_uuid_a, &entry)); + cl_assert_equal_i(entry.granted_mask, 0); + cl_assert_equal_i(entry.declared_mask, APP_PERMISSION_BIT(AppPermission_Microphone)); +} + +void test_app_permissions_db__rejects_invalid_records(void) { + // Non-UUID key + cl_assert_equal_i(E_INVALID_ARGUMENT, app_permissions_db_insert((const uint8_t *)"abc", 3, + (const uint8_t *)&s_granted_mic, + sizeof(s_granted_mic))); + // Short value + cl_assert_equal_i( + E_INVALID_ARGUMENT, + app_permissions_db_insert((const uint8_t *)&s_uuid_a, UUID_SIZE, + (const uint8_t *)&s_granted_mic, sizeof(s_granted_mic) - 1)); + // Wrong version + AppPermissionsDBEntry bad = s_granted_mic; + bad.version = 7; + cl_assert_equal_i(E_INVALID_ARGUMENT, + app_permissions_db_insert((const uint8_t *)&s_uuid_a, UUID_SIZE, + (const uint8_t *)&bad, sizeof(bad))); + cl_assert_equal_i(0, app_permissions_db_get_len((const uint8_t *)&s_uuid_a, UUID_SIZE)); + cl_assert_equal_i(0, app_permissions_db_get_len((const uint8_t *)"abc", 3)); + + AppPermissionsDBEntry entry; + cl_assert_equal_i(E_DOES_NOT_EXIST, app_permissions_db_get(&s_uuid_a, &entry)); +} + +void test_app_permissions_db__longer_value_is_truncated_to_known_layout(void) { + // A newer phone may append fields; we only keep what this firmware understands. + uint8_t buf[sizeof(AppPermissionsDBEntry) + 4]; + memcpy(buf, &s_granted_mic, sizeof(s_granted_mic)); + memset(buf + sizeof(s_granted_mic), 0xAB, 4); + cl_assert_equal_i(S_SUCCESS, app_permissions_db_insert((const uint8_t *)&s_uuid_b, UUID_SIZE, buf, + sizeof(buf))); + cl_assert_equal_i(sizeof(AppPermissionsDBEntry), + app_permissions_db_get_len((const uint8_t *)&s_uuid_b, UUID_SIZE)); + AppPermissionsDBEntry entry; + cl_assert_equal_i(S_SUCCESS, app_permissions_db_get(&s_uuid_b, &entry)); + cl_assert_equal_m((void *)&entry, (void *)&s_granted_mic, sizeof(entry)); +} + +void test_app_permissions_db__set_and_delete_go_through_blob_db(void) { + cl_assert_equal_i(S_SUCCESS, app_permissions_db_set(&s_uuid_a, &s_granted_mic)); + cl_assert_equal_i(1, s_num_inserts); + AppPermissionsDBEntry entry; + cl_assert_equal_i(S_SUCCESS, app_permissions_db_get(&s_uuid_a, &entry)); + + cl_assert_equal_i(S_SUCCESS, app_permissions_db_delete_for_uuid(&s_uuid_a)); + cl_assert_equal_i(1, s_num_deletes); + cl_assert_equal_i(E_DOES_NOT_EXIST, app_permissions_db_get(&s_uuid_a, &entry)); + + cl_assert_equal_i(E_INVALID_ARGUMENT, app_permissions_db_delete((const uint8_t *)"abc", 3)); +} + +static bool prv_count_records(const Uuid *uuid, const AppPermissionsDBEntry *entry, void *context) { + int *count = context; + (*count)++; + cl_assert(uuid_equal(uuid, &s_uuid_a) || uuid_equal(uuid, &s_uuid_b)); + cl_assert_equal_i(entry->version, APP_PERMISSIONS_DB_ENTRY_VERSION); + return true; +} + +static bool prv_stop_after_first(const Uuid *uuid, const AppPermissionsDBEntry *entry, + void *context) { + int *count = context; + (*count)++; + return false; +} + +void test_app_permissions_db__each(void) { + int count = 0; + cl_assert_equal_i(S_SUCCESS, app_permissions_db_each(prv_count_records, &count)); + cl_assert_equal_i(0, count); + + cl_assert_equal_i(S_SUCCESS, app_permissions_db_set(&s_uuid_a, &s_granted_mic)); + cl_assert_equal_i(S_SUCCESS, app_permissions_db_set(&s_uuid_b, &s_denied_mic)); + cl_assert_equal_i(S_SUCCESS, app_permissions_db_each(prv_count_records, &count)); + cl_assert_equal_i(2, count); + + count = 0; + cl_assert_equal_i(S_SUCCESS, app_permissions_db_each(prv_stop_after_first, &count)); + cl_assert_equal_i(1, count); +} + +void test_app_permissions_db__flush(void) { + cl_assert_equal_i(S_SUCCESS, app_permissions_db_set(&s_uuid_a, &s_granted_mic)); + cl_assert_equal_i(S_SUCCESS, app_permissions_db_flush()); + AppPermissionsDBEntry entry; + cl_assert_equal_i(E_DOES_NOT_EXIST, app_permissions_db_get(&s_uuid_a, &entry)); + // Still usable after a flush + cl_assert_equal_i(S_SUCCESS, app_permissions_db_set(&s_uuid_a, &s_denied_mic)); + cl_assert_equal_i(S_SUCCESS, app_permissions_db_get(&s_uuid_a, &entry)); + cl_assert_equal_i(entry.granted_mask, 0); +} diff --git a/tests/fw/services/blob_db/test_blob_db2_endpoint.c b/tests/fw/services/blob_db/test_blob_db2_endpoint.c index d728139dd2..ba9e6a7697 100644 --- a/tests/fw/services/blob_db/test_blob_db2_endpoint.c +++ b/tests/fw/services/blob_db/test_blob_db2_endpoint.c @@ -29,6 +29,7 @@ #include "stubs_prefs_db.h" #include "stubs_reminder_db.h" #include "stubs_watch_app_prefs_db.h" +#include "stubs_app_permissions_db.h" #include "stubs_weather_db.h" #include "stubs_bt_lock.h" #include "stubs_evented_timer.h" diff --git a/tests/fw/services/blob_db/test_blob_db_endpoint.c b/tests/fw/services/blob_db/test_blob_db_endpoint.c index 5f2f3681a5..e51bf75b4b 100644 --- a/tests/fw/services/blob_db/test_blob_db_endpoint.c +++ b/tests/fw/services/blob_db/test_blob_db_endpoint.c @@ -28,6 +28,7 @@ #include "stubs_prefs_db.h" #include "stubs_reminder_db.h" #include "stubs_watch_app_prefs_db.h" +#include "stubs_app_permissions_db.h" #include "stubs_weather_db.h" #include "stubs_health_db.h" #include "stubs_app_glance_db.h" diff --git a/tests/fw/services/test_app_install_manager.c b/tests/fw/services/test_app_install_manager.c index 0d4631850f..3eb5fcdbaf 100644 --- a/tests/fw/services/test_app_install_manager.c +++ b/tests/fw/services/test_app_install_manager.c @@ -28,6 +28,7 @@ // Stub Includes //////////////////////////////////// #include "stubs_analytics.h" +#include "stubs_app_permissions_db.h" #include "stubs_app_manager.h" #include "stubs_app_state.h" #include "stubs_bootbits.h" diff --git a/tests/fw/services/test_app_menu_data_source.c b/tests/fw/services/test_app_menu_data_source.c index 5e9888d369..b5ba980286 100644 --- a/tests/fw/services/test_app_menu_data_source.c +++ b/tests/fw/services/test_app_menu_data_source.c @@ -56,6 +56,7 @@ #include "stubs_pbl_malloc.h" #include "stubs_pebble_tasks.h" #include "stubs_persist.h" +#include "stubs_app_permissions_db.h" #include "stubs_pin_db.h" #include "stubs_process_loader.h" #include "stubs_process_manager.h" diff --git a/tests/stubs/stubs_app_permissions_db.h b/tests/stubs/stubs_app_permissions_db.h new file mode 100644 index 0000000000..57ce36316f --- /dev/null +++ b/tests/stubs/stubs_app_permissions_db.h @@ -0,0 +1,55 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "pbl/services/blob_db/app_permissions_db.h" + +status_t app_permissions_db_get(const Uuid *uuid, AppPermissionsDBEntry *entry_out) { + return E_DOES_NOT_EXIST; +} + +status_t app_permissions_db_set(const Uuid *uuid, const AppPermissionsDBEntry *entry) { + return S_SUCCESS; +} + +status_t app_permissions_db_delete_for_uuid(const Uuid *uuid) { + return S_SUCCESS; +} + +status_t app_permissions_db_each(AppPermissionsDBEachCallback cb, void *context) { + return S_SUCCESS; +} + +/////////////////////////////////////////// +// BlobDB Boilerplate (see blob_db/api.h) +/////////////////////////////////////////// + +void app_permissions_db_init(void) { +} + +status_t app_permissions_db_insert(const uint8_t *key, int key_len, const uint8_t *val, + int val_len) { + return S_SUCCESS; +} + +int app_permissions_db_get_len(const uint8_t *key, int key_len) { + return 0; +} + +status_t app_permissions_db_read(const uint8_t *key, int key_len, uint8_t *val_out, + int val_out_len) { + return S_SUCCESS; +} + +status_t app_permissions_db_delete(const uint8_t *key, int key_len) { + return S_SUCCESS; +} + +status_t app_permissions_db_flush(void) { + return S_SUCCESS; +} + +status_t app_permissions_db_compact(void) { + return S_SUCCESS; +} From 8e81aba7ae82df14043ef30f1614404965799c31 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:32:53 +0200 Subject: [PATCH 03/15] services: add app_permissions service Turns the phone-pushed grant records and the app's manifest declaration into a per-app AppPermissionState (not declared / denied / granted). System apps are always granted; declared apps without a record are denied, except on SDK shell builds where sideloaded apps default to granted so development works without the phone. When the running app's record changes the service emits PEBBLE_APP_PERMISSION_EVENT so the app can react. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012ekdUsEpAe9dNq5wyLYhGc Signed-off-by: Joshua Jun --- .../app_permissions/app_permissions.h | 40 +++ src/fw/kernel/events.h | 7 + src/fw/prj_normal.conf | 1 + src/fw/services/CMakeLists.txt | 1 + src/fw/services/Kconfig | 1 + .../services/app_permissions/CMakeLists.txt | 5 + src/fw/services/app_permissions/Kconfig | 17 ++ .../app_permissions/app_permissions.c | 131 +++++++++ src/fw/services/services_normal/service.c | 5 + tests/fw/services/CMakeLists.txt | 7 + tests/fw/services/test_app_permissions.c | 259 ++++++++++++++++++ 11 files changed, 474 insertions(+) create mode 100644 include/pbl/services/app_permissions/app_permissions.h create mode 100644 src/fw/services/app_permissions/CMakeLists.txt create mode 100644 src/fw/services/app_permissions/Kconfig create mode 100644 src/fw/services/app_permissions/app_permissions.c create mode 100644 tests/fw/services/test_app_permissions.c diff --git a/include/pbl/services/app_permissions/app_permissions.h b/include/pbl/services/app_permissions/app_permissions.h new file mode 100644 index 0000000000..83af52c511 --- /dev/null +++ b/include/pbl/services/app_permissions/app_permissions.h @@ -0,0 +1,40 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "pbl/services/app_permissions/app_permissions_types.h" +#include "pbl/util/uuid.h" +#include "system/status_codes.h" + +#include + +//! Kernel-side view of per-app permission grants. +//! +//! The phone is the source of truth and pushes grant records into BlobDBIdAppPermissions. This +//! service turns those records plus the app's manifest declaration into an AppPermissionState, +//! and notifies the running app (PEBBLE_APP_PERMISSION_EVENT) when its grants change. +//! +//! Rules: +//! - Not declared in the app's manifest -> NotDeclared, whatever the record says. +//! - System apps -> Granted. +//! - Declared but no record (e.g. old phone app) -> Denied. SDK shell builds default to Granted +//! so sideloaded apps work without the phone. + +//! Subscribes to BlobDB events. Call once at boot after blob_db_init_dbs(). +void app_permissions_init(void); + +//! @param uuid The app's UUID +//! @param declared Whether the app declares the permission in its manifest +//! @param permission The permission to look up +AppPermissionState app_permissions_get_state_for_app(const Uuid *uuid, bool declared, + AppPermission permission); + +//! State of a permission for the app currently running in the app task (NotDeclared if none). +AppPermissionState app_permissions_get_state_for_current_app(AppPermission permission); + +bool app_permissions_is_granted_for_current_app(AppPermission permission); + +//! Grants or revokes a permission locally (console / test tooling). Writes through the BlobDB so +//! the same events fire as for a phone-originated change. +status_t app_permissions_set_granted(const Uuid *uuid, AppPermission permission, bool granted); diff --git a/src/fw/kernel/events.h b/src/fw/kernel/events.h index 5b3788ba1b..ddd19ea440 100644 --- a/src/fw/kernel/events.h +++ b/src/fw/kernel/events.h @@ -128,6 +128,7 @@ typedef enum { PEBBLE_PREF_CHANGE_EVENT, PEBBLE_SPEAKER_EVENT, PEBBLE_BACKLIGHT_EVENT, + PEBBLE_APP_PERMISSION_EVENT, PEBBLE_NUM_EVENTS } PebbleEventType; @@ -498,6 +499,11 @@ typedef struct PACKED { // 1 byte bool is_on; } PebbleBacklightEvent; +typedef struct PACKED { // 2 bytes + uint8_t permission; //!< AppPermission + uint8_t state; //!< AppPermissionState +} PebbleAppPermissionEvent; + typedef enum { VoiceEventTypeSessionSetup, VoiceEventTypeSessionResult, @@ -808,6 +814,7 @@ typedef struct PACKED { PebblePrefChangeEvent pref_change; PebbleSpeakerEvent speaker; PebbleBacklightEvent backlight; + PebbleAppPermissionEvent app_permission; }; PebbleTaskBitset task_mask; // 1 == filter out, 0 == leave in // NOTE: we put this 8 bit field at the end so that we can pack this structure and still keep the diff --git a/src/fw/prj_normal.conf b/src/fw/prj_normal.conf index fa21f0b408..6643801d08 100644 --- a/src/fw/prj_normal.conf +++ b/src/fw/prj_normal.conf @@ -7,6 +7,7 @@ CONFIG_SERVICE_ALARMS=y CONFIG_SERVICE_APP_CACHE=y CONFIG_SERVICE_APP_FETCH_ENDPOINT=y CONFIG_SERVICE_APP_GLANCES=y +CONFIG_SERVICE_APP_PERMISSIONS=y CONFIG_SERVICE_APP_INBOX_SERVICE=y CONFIG_SERVICE_APP_MESSAGE=y CONFIG_SERVICE_APP_ORDER_ENDPOINT=y diff --git a/src/fw/services/CMakeLists.txt b/src/fw/services/CMakeLists.txt index bbf8e76c84..6972cf833f 100644 --- a/src/fw/services/CMakeLists.txt +++ b/src/fw/services/CMakeLists.txt @@ -9,6 +9,7 @@ pbl_add_subdirectory_ifdef(CONFIG_SERVICE_ANIMATION_SERVICE animation_service) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_CACHE app_cache) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_FETCH_ENDPOINT app_fetch_endpoint) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_GLANCES app_glances) +pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_PERMISSIONS app_permissions) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_INBOX_SERVICE app_inbox_service) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_MESSAGE app_message) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_ORDER_ENDPOINT app_order_endpoint) diff --git a/src/fw/services/Kconfig b/src/fw/services/Kconfig index 6d01ddd42f..b8f1ebf641 100644 --- a/src/fw/services/Kconfig +++ b/src/fw/services/Kconfig @@ -11,6 +11,7 @@ rsource "animation_service/Kconfig" rsource "app_cache/Kconfig" rsource "app_fetch_endpoint/Kconfig" rsource "app_glances/Kconfig" +rsource "app_permissions/Kconfig" rsource "app_inbox_service/Kconfig" rsource "app_message/Kconfig" rsource "app_order_endpoint/Kconfig" diff --git a/src/fw/services/app_permissions/CMakeLists.txt b/src/fw/services/app_permissions/CMakeLists.txt new file mode 100644 index 0000000000..ee6da1422d --- /dev/null +++ b/src/fw/services/app_permissions/CMakeLists.txt @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +pbl_library() +pbl_library_sources(app_permissions.c) diff --git a/src/fw/services/app_permissions/Kconfig b/src/fw/services/app_permissions/Kconfig new file mode 100644 index 0000000000..72d9b15d15 --- /dev/null +++ b/src/fw/services/app_permissions/Kconfig @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +config SERVICE_APP_PERMISSIONS + bool "App permissions" + depends on SERVICE_BLOB_DB + help + Per-app permission grants (microphone, ...) pushed by the phone and + enforced by the firmware. + +if SERVICE_APP_PERMISSIONS + +module = SERVICE_APP_PERMISSIONS +module-str = App permissions +source "subsys/logging/Kconfig.template.log_level" + +endif diff --git a/src/fw/services/app_permissions/app_permissions.c b/src/fw/services/app_permissions/app_permissions.c new file mode 100644 index 0000000000..1482378f8e --- /dev/null +++ b/src/fw/services/app_permissions/app_permissions.c @@ -0,0 +1,131 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "pbl/services/app_permissions/app_permissions.h" + +#include "applib/event_service_client.h" +#include "kernel/events.h" +#include "pbl/services/blob_db/api.h" +#include "pbl/services/blob_db/app_permissions_db.h" +#include "process_management/app_install_manager.h" +#include "process_management/app_manager.h" +#include "process_management/pebble_process_md.h" +#include + +#ifdef CONFIG_SERVICE_MIC_CAPTURE +#include "pbl/services/mic_capture/mic_capture_service.h" +#endif + +PBL_LOG_MODULE_DEFINE(service_app_permissions, CONFIG_SERVICE_APP_PERMISSIONS_LOG_LEVEL); + +static bool prv_md_declares(const PebbleProcessMd *md, AppPermission permission) { + switch (permission) { + case AppPermission_Microphone: + // Watchfaces run unattended; they never get the microphone + return md->uses_microphone && (md->process_type != ProcessTypeWatchface); + case AppPermissionCount: + break; + } + return false; +} + +AppPermissionState app_permissions_get_state_for_app(const Uuid *uuid, bool declared, + AppPermission permission) { + if (!uuid || (permission >= AppPermissionCount) || !declared) { + return AppPermissionStateNotDeclared; + } + + AppPermissionsDBEntry entry; + if (app_permissions_db_get(uuid, &entry) != S_SUCCESS) { +#ifdef CONFIG_SHELL_SDK + // No phone in the loop: let sideloaded apps use what they declare. + return AppPermissionStateGranted; +#else + return AppPermissionStateDenied; +#endif + } + + return (entry.granted_mask & APP_PERMISSION_BIT(permission)) ? AppPermissionStateGranted + : AppPermissionStateDenied; +} + +AppPermissionState app_permissions_get_state_for_current_app(AppPermission permission) { + const PebbleProcessMd *md = app_manager_get_current_app_md(); + if (!md || (permission >= AppPermissionCount)) { + return AppPermissionStateNotDeclared; + } + if (app_install_id_from_system(app_manager_get_current_app_id())) { + return AppPermissionStateGranted; + } + return app_permissions_get_state_for_app(&md->uuid, prv_md_declares(md, permission), permission); +} + +bool app_permissions_is_granted_for_current_app(AppPermission permission) { + return (app_permissions_get_state_for_current_app(permission) == AppPermissionStateGranted); +} + +status_t app_permissions_set_granted(const Uuid *uuid, AppPermission permission, bool granted) { + if (!uuid || (permission >= AppPermissionCount)) { + return E_INVALID_ARGUMENT; + } + AppPermissionsDBEntry entry; + if (app_permissions_db_get(uuid, &entry) != S_SUCCESS) { + entry = (AppPermissionsDBEntry){.version = APP_PERMISSIONS_DB_ENTRY_VERSION}; + } + const AppPermissionMask bit = APP_PERMISSION_BIT(permission); + entry.declared_mask |= bit; + if (granted) { + entry.granted_mask |= bit; + } else { + entry.granted_mask &= ~bit; + } + return app_permissions_db_set(uuid, &entry); +} + +static void prv_notify_current_app(void) { + for (AppPermission permission = 0; permission < AppPermissionCount; permission++) { + const AppPermissionState state = app_permissions_get_state_for_current_app(permission); + if (state == AppPermissionStateNotDeclared) { + continue; + } + PBL_LOG_DBG("Permission %u for current app is now %u", permission, state); + PebbleEvent event = { + .type = PEBBLE_APP_PERMISSION_EVENT, + .app_permission = { + .permission = permission, + .state = state, + }, + }; + event_put(&event); + } +#ifdef CONFIG_SERVICE_MIC_CAPTURE + mic_capture_service_handle_permission_changed(); +#endif +} + +static void prv_blob_db_event_handler(PebbleEvent *event, void *context) { + const PebbleBlobDBEvent *blob_db_event = &event->blob_db; + if (blob_db_event->db_id != BlobDBIdAppPermissions) { + return; + } + + const PebbleProcessMd *md = app_manager_get_current_app_md(); + if (!md) { + return; + } + + const bool affects_current_app = (blob_db_event->type == BlobDBEventTypeFlush) || + ((blob_db_event->key_len == UUID_SIZE) && + uuid_equal((const Uuid *)blob_db_event->key, &md->uuid)); + if (affects_current_app) { + prv_notify_current_app(); + } +} + +void app_permissions_init(void) { + static EventServiceInfo s_blob_db_event_info = { + .type = PEBBLE_BLOBDB_EVENT, + .handler = prv_blob_db_event_handler, + }; + event_service_client_subscribe(&s_blob_db_event_info); +} diff --git a/src/fw/services/services_normal/service.c b/src/fw/services/services_normal/service.c index 7bad4aa5fa..2d6bf84f56 100644 --- a/src/fw/services/services_normal/service.c +++ b/src/fw/services/services_normal/service.c @@ -13,6 +13,7 @@ #include "pbl/services/alarms/alarm.h" #include "pbl/services/app_cache.h" #include "pbl/services/app_glances/app_glance_service.h" +#include "pbl/services/app_permissions/app_permissions.h" #include "pbl/services/blob_db/api.h" #include "pbl/services/blob_db/endpoint_private.h" #include "pbl/services/data_logging/data_logging_service.h" @@ -125,6 +126,10 @@ void services_normal_init(void) { #endif app_glance_service_init(); + +#ifdef CONFIG_SERVICE_APP_PERMISSIONS + app_permissions_init(); +#endif } static struct ServiceRunLevelSetting s_runlevel_settings[] = { diff --git a/tests/fw/services/CMakeLists.txt b/tests/fw/services/CMakeLists.txt index ad22a9f1b8..0147219371 100644 --- a/tests/fw/services/CMakeLists.txt +++ b/tests/fw/services/CMakeLists.txt @@ -305,6 +305,13 @@ pbl_clar_test(test_app_cache OVERRIDES dummy_board ) +pbl_clar_test(test_app_permissions + SOURCES + src/fw/services/app_permissions/app_permissions.c + tests/fakes/fake_events.c + OVERRIDES dummy_board +) + pbl_clar_test(test_app_install_manager SOURCES src/fw/flash_region/flash_region.c diff --git a/tests/fw/services/test_app_permissions.c b/tests/fw/services/test_app_permissions.c new file mode 100644 index 0000000000..26eee0d64c --- /dev/null +++ b/tests/fw/services/test_app_permissions.c @@ -0,0 +1,259 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "clar.h" + +#include "applib/event_service_client.h" +#include "kernel/events.h" +#include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/blob_db/api.h" +#include "pbl/services/blob_db/app_permissions_db.h" +#include "process_management/app_install_manager.h" +#include "process_management/app_manager.h" +#include "process_management/pebble_process_md.h" + +#include + +// Fakes +//////////////////////////////////////////////////////////////// +#include "fake_events.h" + +// Stubs +//////////////////////////////////////////////////////////////// +#include "stubs_logging.h" +#include "stubs_passert.h" +#include "stubs_pbl_malloc.h" + +// In-memory single-record fake of the AppPermissions BlobDB +//////////////////////////////////////////////////////////////// + +static bool s_has_record; +static Uuid s_record_uuid; +static AppPermissionsDBEntry s_record; +static int s_num_sets; + +status_t app_permissions_db_get(const Uuid *uuid, AppPermissionsDBEntry *entry_out) { + if (!s_has_record || !uuid_equal(uuid, &s_record_uuid)) { + return E_DOES_NOT_EXIST; + } + *entry_out = s_record; + return S_SUCCESS; +} + +status_t app_permissions_db_set(const Uuid *uuid, const AppPermissionsDBEntry *entry) { + s_has_record = true; + s_record_uuid = *uuid; + s_record = *entry; + s_num_sets++; + return S_SUCCESS; +} + +// Current app +//////////////////////////////////////////////////////////////// + +static PebbleProcessMd s_md; +static const PebbleProcessMd *s_current_md; +static AppInstallId s_current_id = 1; + +const PebbleProcessMd *app_manager_get_current_app_md(void) { + return s_current_md; +} + +AppInstallId app_manager_get_current_app_id(void) { + return s_current_id; +} + +bool app_install_id_from_system(AppInstallId id) { + return (id < INSTALL_ID_INVALID); +} + +// Event service +//////////////////////////////////////////////////////////////// + +static EventServiceInfo *s_blob_db_subscription; + +void event_service_client_subscribe(EventServiceInfo *info) { + cl_assert_equal_i(info->type, PEBBLE_BLOBDB_EVENT); + s_blob_db_subscription = info; +} + +void event_service_client_unsubscribe(EventServiceInfo *info) { +} + +static void prv_send_blob_db_event(BlobDBId db_id, BlobDBEventType type, const Uuid *key) { + PebbleEvent event = { + .type = PEBBLE_BLOBDB_EVENT, + .blob_db = { + .db_id = db_id, + .type = type, + .key = (uint8_t *)key, + .key_len = key ? UUID_SIZE : 0, + }, + }; + s_blob_db_subscription->handler(&event, s_blob_db_subscription->context); +} + +static const Uuid s_uuid_app = {0x1e, 0xb1, 0xd3, 0x9b, 0x56, 0x98, 0x48, 0x44, + 0xb3, 0x94, 0x1f, 0x87, 0xb6, 0xbe, 0xae, 0x67}; +static const Uuid s_uuid_other = {0xb8, 0x26, 0x2e, 0x08, 0x57, 0xe9, 0x4e, 0x58, + 0x88, 0x02, 0x45, 0xfd, 0xfe, 0xe0, 0xac, 0x77}; + +static void prv_set_record(const Uuid *uuid, AppPermissionMask granted) { + s_has_record = true; + s_record_uuid = *uuid; + s_record = (AppPermissionsDBEntry){ + .version = APP_PERMISSIONS_DB_ENTRY_VERSION, + .granted_mask = granted, + .declared_mask = APP_PERMISSION_BIT(AppPermission_Microphone), + }; +} + +// Setup +//////////////////////////////////////////////////////////////// + +void test_app_permissions__initialize(void) { + fake_event_init(); + s_has_record = false; + s_num_sets = 0; + s_md = (PebbleProcessMd){.uuid = s_uuid_app, .uses_microphone = true}; + s_current_md = &s_md; + s_current_id = 1; + s_blob_db_subscription = NULL; + app_permissions_init(); + cl_assert(s_blob_db_subscription != NULL); +} + +void test_app_permissions__cleanup(void) { +} + +// Tests +//////////////////////////////////////////////////////////////// + +void test_app_permissions__state_matrix(void) { + // Not declared beats everything + prv_set_record(&s_uuid_app, APP_PERMISSION_BIT(AppPermission_Microphone)); + cl_assert_equal_i( + AppPermissionStateNotDeclared, + app_permissions_get_state_for_app(&s_uuid_app, false, AppPermission_Microphone)); + + // Declared + granted record + cl_assert_equal_i(AppPermissionStateGranted, + app_permissions_get_state_for_app(&s_uuid_app, true, AppPermission_Microphone)); + + // Declared + record without the bit + prv_set_record(&s_uuid_app, 0); + cl_assert_equal_i(AppPermissionStateDenied, + app_permissions_get_state_for_app(&s_uuid_app, true, AppPermission_Microphone)); + + // Declared + no record: fail closed + s_has_record = false; + cl_assert_equal_i(AppPermissionStateDenied, + app_permissions_get_state_for_app(&s_uuid_app, true, AppPermission_Microphone)); + + // Out of range permission + cl_assert_equal_i(AppPermissionStateNotDeclared, + app_permissions_get_state_for_app(&s_uuid_app, true, AppPermissionCount)); +} + +void test_app_permissions__current_app(void) { + // No app running + s_current_md = NULL; + cl_assert_equal_i(AppPermissionStateNotDeclared, + app_permissions_get_state_for_current_app(AppPermission_Microphone)); + s_current_md = &s_md; + + // Declared, no record + cl_assert_equal_i(AppPermissionStateDenied, + app_permissions_get_state_for_current_app(AppPermission_Microphone)); + cl_assert_equal_b(false, app_permissions_is_granted_for_current_app(AppPermission_Microphone)); + + // Granted + prv_set_record(&s_uuid_app, APP_PERMISSION_BIT(AppPermission_Microphone)); + cl_assert_equal_b(true, app_permissions_is_granted_for_current_app(AppPermission_Microphone)); + + // A record for a different app does not count + prv_set_record(&s_uuid_other, APP_PERMISSION_BIT(AppPermission_Microphone)); + cl_assert_equal_b(false, app_permissions_is_granted_for_current_app(AppPermission_Microphone)); + + // A watchface never has it, even when declared and granted + s_md.process_type = ProcessTypeWatchface; + cl_assert_equal_i(AppPermissionStateNotDeclared, + app_permissions_get_state_for_current_app(AppPermission_Microphone)); + s_md.process_type = ProcessTypeApp; + + // Not declared in the header + s_md.uses_microphone = false; + prv_set_record(&s_uuid_app, APP_PERMISSION_BIT(AppPermission_Microphone)); + cl_assert_equal_i(AppPermissionStateNotDeclared, + app_permissions_get_state_for_current_app(AppPermission_Microphone)); +} + +void test_app_permissions__system_apps_always_granted(void) { + s_current_id = -5; // system install ids are negative + s_md.uses_microphone = false; + cl_assert_equal_i(AppPermissionStateGranted, + app_permissions_get_state_for_current_app(AppPermission_Microphone)); +} + +void test_app_permissions__set_granted(void) { + cl_assert_equal_i(E_INVALID_ARGUMENT, + app_permissions_set_granted(&s_uuid_app, AppPermissionCount, true)); + cl_assert_equal_i(E_INVALID_ARGUMENT, + app_permissions_set_granted(NULL, AppPermission_Microphone, true)); + + // Creates a record when none exists + cl_assert_equal_i(S_SUCCESS, + app_permissions_set_granted(&s_uuid_app, AppPermission_Microphone, true)); + cl_assert_equal_i(1, s_num_sets); + cl_assert(uuid_equal(&s_record_uuid, &s_uuid_app)); + cl_assert_equal_i(s_record.version, APP_PERMISSIONS_DB_ENTRY_VERSION); + cl_assert_equal_i(s_record.granted_mask, APP_PERMISSION_BIT(AppPermission_Microphone)); + cl_assert_equal_i(s_record.declared_mask, APP_PERMISSION_BIT(AppPermission_Microphone)); + + // Revoke keeps the declared bit + cl_assert_equal_i(S_SUCCESS, + app_permissions_set_granted(&s_uuid_app, AppPermission_Microphone, false)); + cl_assert_equal_i(s_record.granted_mask, 0); + cl_assert_equal_i(s_record.declared_mask, APP_PERMISSION_BIT(AppPermission_Microphone)); +} + +void test_app_permissions__blob_db_event_notifies_current_app(void) { + // Other databases are ignored + prv_send_blob_db_event(BlobDBIdApps, BlobDBEventTypeInsert, &s_uuid_app); + cl_assert_equal_i(0, fake_event_get_count()); + + // Another app's record is ignored + prv_set_record(&s_uuid_other, APP_PERMISSION_BIT(AppPermission_Microphone)); + prv_send_blob_db_event(BlobDBIdAppPermissions, BlobDBEventTypeInsert, &s_uuid_other); + cl_assert_equal_i(0, fake_event_get_count()); + + // Our record: Granted + prv_set_record(&s_uuid_app, APP_PERMISSION_BIT(AppPermission_Microphone)); + prv_send_blob_db_event(BlobDBIdAppPermissions, BlobDBEventTypeInsert, &s_uuid_app); + cl_assert_equal_i(1, fake_event_get_count()); + PebbleEvent e = fake_event_get_last(); + cl_assert_equal_i(e.type, PEBBLE_APP_PERMISSION_EVENT); + cl_assert_equal_i(e.app_permission.permission, AppPermission_Microphone); + cl_assert_equal_i(e.app_permission.state, AppPermissionStateGranted); + + // Delete: back to Denied + s_has_record = false; + prv_send_blob_db_event(BlobDBIdAppPermissions, BlobDBEventTypeDelete, &s_uuid_app); + cl_assert_equal_i(2, fake_event_get_count()); + e = fake_event_get_last(); + cl_assert_equal_i(e.app_permission.state, AppPermissionStateDenied); + + // Flush affects everyone + prv_send_blob_db_event(BlobDBIdAppPermissions, BlobDBEventTypeFlush, NULL); + cl_assert_equal_i(3, fake_event_get_count()); + + // Undeclared apps never hear about it + s_md.uses_microphone = false; + prv_send_blob_db_event(BlobDBIdAppPermissions, BlobDBEventTypeInsert, &s_uuid_app); + cl_assert_equal_i(3, fake_event_get_count()); + + // No app running + s_current_md = NULL; + prv_send_blob_db_event(BlobDBIdAppPermissions, BlobDBEventTypeFlush, NULL); + cl_assert_equal_i(3, fake_event_get_count()); +} From 6520ab911ffc37f19d6f201bf24263c73af30da9 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:36:17 +0200 Subject: [PATCH 04/15] applib: add app permissions API Lets apps query the state of a declared permission and subscribe to changes (app_permission_get_state, app_permission_is_granted, app_permission_service_subscribe/unsubscribe), exported as SDK revision 110. Also advertises app_permissions_support (capability bit 25) to the phone so it knows the watch accepts grant records; enforcement does not depend on the bit. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- .../app_permissions/app_permissions.h | 1 + .../app_permissions/app_permissions_types.h | 24 ------- .../pbl/services/blob_db/app_permissions_db.h | 17 +++++ include/pbl/services/comm_session/session.h | 1 + .../comm_session/session_remote_version.h | 5 +- src/fw/applib/app_permissions.c | 67 +++++++++++++++++++ src/fw/applib/app_permissions.h | 45 +++++++++++++ src/fw/applib/app_permissions_private.h | 15 +++++ src/fw/kernel/system_versions.c | 3 + .../process_management/pebble_process_info.h | 3 +- src/fw/process_state/app_state/app_state.c | 8 +++ src/fw/process_state/app_state/app_state.h | 3 + src/fw/syscall/syscall.h | 3 + src/fw/syscall/syscall_app_permissions.c | 19 ++++++ .../generate_native_sdk/exported_symbols.json | 39 ++++++++++- 15 files changed, 224 insertions(+), 29 deletions(-) create mode 100644 src/fw/applib/app_permissions.c create mode 100644 src/fw/applib/app_permissions.h create mode 100644 src/fw/applib/app_permissions_private.h create mode 100644 src/fw/syscall/syscall_app_permissions.c diff --git a/include/pbl/services/app_permissions/app_permissions.h b/include/pbl/services/app_permissions/app_permissions.h index 83af52c511..249d997a05 100644 --- a/include/pbl/services/app_permissions/app_permissions.h +++ b/include/pbl/services/app_permissions/app_permissions.h @@ -4,6 +4,7 @@ #pragma once #include "pbl/services/app_permissions/app_permissions_types.h" +#include "pbl/services/blob_db/app_permissions_db.h" #include "pbl/util/uuid.h" #include "system/status_codes.h" diff --git a/include/pbl/services/app_permissions/app_permissions_types.h b/include/pbl/services/app_permissions/app_permissions_types.h index 075011d93a..3b73a2f89e 100644 --- a/include/pbl/services/app_permissions/app_permissions_types.h +++ b/include/pbl/services/app_permissions/app_permissions_types.h @@ -3,10 +3,6 @@ #pragma once -#include "pbl/util/attributes.h" - -#include - //! @addtogroup Foundation //! @{ //! @addtogroup Permissions @@ -31,23 +27,3 @@ typedef enum AppPermissionState { //! @} // end addtogroup Permissions //! @} // end addtogroup Foundation - -//! @internal -typedef uint32_t AppPermissionMask; - -//! @internal -#define APP_PERMISSION_BIT(permission) ((AppPermissionMask)1u << (permission)) - -//! @internal -#define APP_PERMISSIONS_DB_ENTRY_VERSION (1) - -//! @internal -//! Serialized grant record pushed by the phone (BlobDBIdAppPermissions, keyed by app Uuid). -typedef struct PACKED AppPermissionsDBEntry { - uint8_t version; //!< APP_PERMISSIONS_DB_ENTRY_VERSION - uint8_t reserved[3]; - AppPermissionMask granted_mask; //!< Permissions the user granted - AppPermissionMask declared_mask; //!< Permissions the app declared (informational) -} AppPermissionsDBEntry; - -_Static_assert(sizeof(AppPermissionsDBEntry) == 12, "AppPermissionsDBEntry layout changed"); diff --git a/include/pbl/services/blob_db/app_permissions_db.h b/include/pbl/services/blob_db/app_permissions_db.h index 09bac1bd58..5a862b09ae 100644 --- a/include/pbl/services/blob_db/app_permissions_db.h +++ b/include/pbl/services/blob_db/app_permissions_db.h @@ -4,12 +4,29 @@ #pragma once #include "pbl/services/app_permissions/app_permissions_types.h" +#include "pbl/util/attributes.h" #include "pbl/util/uuid.h" #include "system/status_codes.h" #include #include +typedef uint32_t AppPermissionMask; + +#define APP_PERMISSION_BIT(permission) ((AppPermissionMask)1u << (permission)) + +#define APP_PERMISSIONS_DB_ENTRY_VERSION (1) + +//! Serialized grant record pushed by the phone (BlobDBIdAppPermissions, keyed by app Uuid). +typedef struct PACKED AppPermissionsDBEntry { + uint8_t version; //!< APP_PERMISSIONS_DB_ENTRY_VERSION + uint8_t reserved[3]; + AppPermissionMask granted_mask; //!< Permissions the user granted + AppPermissionMask declared_mask; //!< Permissions the app declared (informational) +} AppPermissionsDBEntry; + +_Static_assert(sizeof(AppPermissionsDBEntry) == 12, "AppPermissionsDBEntry layout changed"); + //! Per-app permission grants, pushed by the phone and keyed by app Uuid. //! Reads the grant record for an app. diff --git a/include/pbl/services/comm_session/session.h b/include/pbl/services/comm_session/session.h index d60d9ec217..da9094d86e 100644 --- a/include/pbl/services/comm_session/session.h +++ b/include/pbl/services/comm_session/session.h @@ -48,6 +48,7 @@ typedef enum { CommSessionSmoothFwInstallProgressSupport = 1 << 14, CommSessionImagingSupport = 1 << 17, CommSessionSettingsSyncSupport = 1 << 23, + CommSessionAppPermissionsSupport = 1 << 25, CommSessionOutOfRange } CommSessionCapability; diff --git a/include/pbl/services/comm_session/session_remote_version.h b/include/pbl/services/comm_session/session_remote_version.h index f9aee8080b..4bf317ef55 100644 --- a/include/pbl/services/comm_session/session_remote_version.h +++ b/include/pbl/services/comm_session/session_remote_version.h @@ -38,8 +38,9 @@ typedef struct PACKED { uint8_t more_padded_bits : 2; bool continue_fw_install_across_disconnect_support : 1; bool blob_db_version_support : 1; - bool settings_sync_support : 1; // Phone supports Settings BlobDB sync - bool weather_db_v4_support : 1; // Phone writes the v4 weather BlobDB record (rich forecast) + bool settings_sync_support : 1; // Phone supports Settings BlobDB sync + bool weather_db_v4_support : 1; // Phone writes the v4 weather BlobDB record (rich forecast) + bool app_permissions_support : 1; // Phone pushes per-app permission grants (BlobDB 0x0D) }; uint64_t flags; }; diff --git a/src/fw/applib/app_permissions.c b/src/fw/applib/app_permissions.c new file mode 100644 index 0000000000..df24a1f224 --- /dev/null +++ b/src/fw/applib/app_permissions.c @@ -0,0 +1,67 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "applib/app_permissions.h" +#include "applib/app_permissions_private.h" + +#include "kernel/events.h" +#include "kernel/pebble_tasks.h" +#include "process_state/app_state/app_state.h" +#include "syscall/syscall.h" + +static AppPermissionServiceState *prv_get_state(void) { + // Workers have no permissions of their own. + if (pebble_task_get_current() != PebbleTask_App) { + return NULL; + } + return app_state_get_app_permission_service_state(); +} + +static void prv_handle_event(PebbleEvent *e, void *context) { + AppPermissionServiceState *state = context; + if (state->handler) { + state->handler((AppPermission)e->app_permission.permission, + (AppPermissionState)e->app_permission.state, state->context); + } +} + +AppPermissionState app_permission_get_state(AppPermission permission) { + if ((permission >= AppPermissionCount) || (pebble_task_get_current() != PebbleTask_App)) { + return AppPermissionStateNotDeclared; + } + return (AppPermissionState)sys_app_permission_get_state((uint8_t)permission); +} + +bool app_permission_is_granted(AppPermission permission) { + return (app_permission_get_state(permission) == AppPermissionStateGranted); +} + +void app_permission_service_subscribe(AppPermissionChangedHandler handler, void *context) { + AppPermissionServiceState *state = prv_get_state(); + if (!state) { + return; + } + state->handler = handler; + state->context = context; + event_service_client_subscribe(&state->event_info); +} + +void app_permission_service_unsubscribe(void) { + AppPermissionServiceState *state = prv_get_state(); + if (!state) { + return; + } + event_service_client_unsubscribe(&state->event_info); + state->handler = NULL; + state->context = NULL; +} + +void app_permission_service_state_init(AppPermissionServiceState *state) { + *state = (AppPermissionServiceState){ + .event_info = { + .type = PEBBLE_APP_PERMISSION_EVENT, + .handler = prv_handle_event, + .context = state, + }, + }; +} diff --git a/src/fw/applib/app_permissions.h b/src/fw/applib/app_permissions.h new file mode 100644 index 0000000000..41d43ee76a --- /dev/null +++ b/src/fw/applib/app_permissions.h @@ -0,0 +1,45 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "pbl/services/app_permissions/app_permissions_types.h" + +#include + +//! @addtogroup Foundation +//! @{ +//! @addtogroup Permissions +//! \brief Querying the permissions the user granted to your app +//! +//! Some capabilities, such as live microphone access, must be declared in your app's manifest +//! (`capabilities: ["microphone"]`) and granted by the user in the phone app. The grant can be +//! revoked at any time. Use this API to check the current state and to be told when it changes. +//! The system enforces the permission independently: an API that needs a permission fails when it +//! is not granted. +//! @{ + +//! Handler called when the state of one of the app's declared permissions changes. +//! @param permission The permission that changed +//! @param state The new state +//! @param context The context passed to \ref app_permission_service_subscribe +typedef void (*AppPermissionChangedHandler)(AppPermission permission, AppPermissionState state, + void *context); + +//! Gets the state of a permission for the running app. +//! @return \ref AppPermissionStateNotDeclared if the app does not declare it in its manifest +AppPermissionState app_permission_get_state(AppPermission permission); + +//! @return true if the permission is declared and granted +bool app_permission_is_granted(AppPermission permission); + +//! Subscribes to permission changes. Only one handler can be registered at a time. +//! @param handler Handler called on the app task whenever a declared permission changes state +//! @param context User-provided context passed to the handler +void app_permission_service_subscribe(AppPermissionChangedHandler handler, void *context); + +//! Unsubscribes from permission changes. +void app_permission_service_unsubscribe(void); + +//! @} // end addtogroup Permissions +//! @} // end addtogroup Foundation diff --git a/src/fw/applib/app_permissions_private.h b/src/fw/applib/app_permissions_private.h new file mode 100644 index 0000000000..7a9bd1076d --- /dev/null +++ b/src/fw/applib/app_permissions_private.h @@ -0,0 +1,15 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "applib/app_permissions.h" +#include "applib/event_service_client.h" + +typedef struct AppPermissionServiceState { + EventServiceInfo event_info; + AppPermissionChangedHandler handler; + void *context; +} AppPermissionServiceState; + +void app_permission_service_state_init(AppPermissionServiceState *state); diff --git a/src/fw/kernel/system_versions.c b/src/fw/kernel/system_versions.c index 46d2da4bf5..7ac55279c2 100644 --- a/src/fw/kernel/system_versions.c +++ b/src/fw/kernel/system_versions.c @@ -141,6 +141,9 @@ static void prv_send_watch_versions(CommSession *session) { versions_msg.capabilities.custom_vibe_pattern_support = 1; versions_msg.capabilities.blob_db_version_support = 1; versions_msg.capabilities.weather_db_v4_support = 1; +#ifdef CONFIG_SERVICE_APP_PERMISSIONS + versions_msg.capabilities.app_permissions_support = 1; +#endif versions_msg.capabilities.notification_image_support = NOTIFICATION_IMAGE_SUPPORTED; bt_local_id_copy_address(&versions_msg.device_address); diff --git a/src/fw/process_management/pebble_process_info.h b/src/fw/process_management/pebble_process_info.h index 93088d03f8..5503c9ca71 100644 --- a/src/fw/process_management/pebble_process_info.h +++ b/src/fw/process_management/pebble_process_info.h @@ -170,9 +170,10 @@ typedef enum { // (tap/pan/swipe + window attach/detach) to apps (rev 107) sdk.major:0x5 .minor:0x69 -- Add // app_touch_navigation_enable() opt-in for third-party touch nav (rev 108) sdk.major:0x5 // .minor:0x6a -- Add HRV sampling API (health_service_set_hrv_sample_period) (rev 109) +// sdk.major:0x5 .minor:0x6b -- Add app permissions API (rev 110) #define PROCESS_INFO_CURRENT_SDK_VERSION_MAJOR 0x5 -#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6a +#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6b // The first SDK to ship with 2.x APIs #define PROCESS_INFO_FIRST_2X_SDK_VERSION_MAJOR 0x4 diff --git a/src/fw/process_state/app_state/app_state.c b/src/fw/process_state/app_state/app_state.c index 0a92b537f3..58213d1d71 100644 --- a/src/fw/process_state/app_state/app_state.c +++ b/src/fw/process_state/app_state/app_state.c @@ -81,6 +81,8 @@ typedef struct { ConnectionServiceState connection_service_state; + AppPermissionServiceState app_permission_service_state; + HealthServiceState health_service_state; LocaleInfo locale_info; @@ -307,6 +309,8 @@ NOINLINE void app_state_init(void) { connection_service_state_init(app_state_get_connection_service_state()); + app_permission_service_state_init(app_state_get_app_permission_service_state()); + tick_timer_service_state_init(app_state_get_tick_timer_service_state()); touch_service_state_init(app_state_get_touch_service_state()); @@ -440,6 +444,10 @@ ConnectionServiceState *app_state_get_connection_service_state(void) { return &s_app_state_ptr->connection_service_state; } +AppPermissionServiceState *app_state_get_app_permission_service_state(void) { + return &s_app_state_ptr->app_permission_service_state; +} + HealthServiceState *app_state_get_health_service_state(void) { return &s_app_state_ptr->health_service_state; } diff --git a/src/fw/process_state/app_state/app_state.h b/src/fw/process_state/app_state/app_state.h index c5cec4a515..72281599a5 100644 --- a/src/fw/process_state/app_state/app_state.h +++ b/src/fw/process_state/app_state/app_state.h @@ -7,6 +7,7 @@ #include "applib/app_focus_service.h" #include "applib/app_inbox.h" #include "applib/app_message/app_message_internal.h" +#include "applib/app_permissions_private.h" #include "applib/app_wakeup.h" #include "applib/backlight_service_private.h" #include "applib/battery_state_service_private.h" @@ -116,6 +117,8 @@ TouchServiceState *app_state_get_touch_service_state(void); ConnectionServiceState *app_state_get_connection_service_state(void); +AppPermissionServiceState *app_state_get_app_permission_service_state(void); + LocaleInfo *app_state_get_locale_info(void); ContentIndicatorsBuffer *app_state_get_content_indicators_buffer(void); diff --git a/src/fw/syscall/syscall.h b/src/fw/syscall/syscall.h index 7fd5c1b18c..8a72ac7dd2 100644 --- a/src/fw/syscall/syscall.h +++ b/src/fw/syscall/syscall.h @@ -103,6 +103,9 @@ uint8_t sys_speaker_get_state(void); void sys_speaker_register_finish(void); bool sys_speaker_is_muted(void); +//! @return AppPermissionState for the running app +uint8_t sys_app_permission_get_state(uint8_t permission); + void sys_get_app_uuid(Uuid *uuid); bool sys_app_is_watchface(void); AppInstallId sys_app_manager_get_current_app_id(void); diff --git a/src/fw/syscall/syscall_app_permissions.c b/src/fw/syscall/syscall_app_permissions.c new file mode 100644 index 0000000000..b24743c064 --- /dev/null +++ b/src/fw/syscall/syscall_app_permissions.c @@ -0,0 +1,19 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "syscall/syscall.h" +#include "syscall/syscall_internal.h" + +#include "pbl/services/app_permissions/app_permissions.h" + +DEFINE_SYSCALL(uint8_t, sys_app_permission_get_state, uint8_t permission) { + if (permission >= AppPermissionCount) { + syscall_failed(); + } +#ifdef CONFIG_SERVICE_APP_PERMISSIONS + return (uint8_t)app_permissions_get_state_for_current_app((AppPermission)permission); +#else + // Builds without the service (e.g. PRF) never grant anything. + return (uint8_t)AppPermissionStateDenied; +#endif +} diff --git a/tools/generate_native_sdk/exported_symbols.json b/tools/generate_native_sdk/exported_symbols.json index 7ea9c42891..55dad92834 100644 --- a/tools/generate_native_sdk/exported_symbols.json +++ b/tools/generate_native_sdk/exported_symbols.json @@ -4,7 +4,7 @@ "You should also make sure you are obeying our API design guidelines:", "https://pebbletechnology.atlassian.net/wiki/display/DEV/SDK+API+Design+Guidelines" ], - "revision" : "109", + "revision" : "110", "version" : "2.0", "files": [ "include/pbl/drivers/ambient_light.h", @@ -135,7 +135,9 @@ "fw/applib/voice/dictation_session.h", "fw/applib/ui/content_indicator.h", "fw/applib/rockyjs/rocky.h", - "fw/applib/unobstructed_area_service.h" + "fw/applib/unobstructed_area_service.h", + "include/pbl/services/app_permissions/app_permissions_types.h", + "fw/applib/app_permissions.h" ], "exports": [ { @@ -4746,6 +4748,39 @@ "addedRevision": "100" } ] + }, { + "type": "group", + "name": "Permissions", + "appOnly": true, + "addedRevision": "110", + "exports": [ + { + "type": "type", + "name": "AppPermission" + }, { + "type": "type", + "name": "AppPermissionState" + }, { + "type": "type", + "name": "AppPermissionChangedHandler" + }, { + "type": "function", + "name": "app_permission_get_state", + "addedRevision": "110" + }, { + "type": "function", + "name": "app_permission_is_granted", + "addedRevision": "110" + }, { + "type": "function", + "name": "app_permission_service_subscribe", + "addedRevision": "110" + }, { + "type": "function", + "name": "app_permission_service_unsubscribe", + "addedRevision": "110" + } + ] }, { "type": "group", "name": "Light", From 262b631dbdbf1b2d66381360c9034c81a5ac7f14 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:38:41 +0200 Subject: [PATCH 05/15] console: add perm prompt commands `perm list` dumps the grant records and whether the connected phone supports pushing them; `perm grant mic` / `perm revoke mic` write a grant locally so permissions can be exercised without the phone. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- src/fw/console/prompt_commands.h | 10 +++ .../services/app_permissions/CMakeLists.txt | 5 +- .../app_permissions_commands.c | 73 +++++++++++++++++++ 3 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 src/fw/services/app_permissions/app_permissions_commands.c diff --git a/src/fw/console/prompt_commands.h b/src/fw/console/prompt_commands.h index 1de334799d..ee3d4f0bec 100644 --- a/src/fw/console/prompt_commands.h +++ b/src/fw/console/prompt_commands.h @@ -128,6 +128,11 @@ extern void command_get_active_app_metadata(void); extern void command_app_list(void); extern void command_app_launch(const char *app_num_str); extern void command_app_remove(const char *app_num_str); +#ifdef CONFIG_SERVICE_APP_PERMISSIONS +extern void command_perm_list(void); +extern void command_perm_grant(const char *app_num_str, const char *permission); +extern void command_perm_revoke(const char *app_num_str, const char *permission); +#endif extern void command_worker_launch(const char *app_num_str); extern void command_worker_kill(void); @@ -344,6 +349,11 @@ static const Command s_prompt_commands[] = { {"app list", command_app_list, 0}, {"app launch", command_app_launch, 1}, {"app remove", command_app_remove, 1}, +#ifdef CONFIG_SERVICE_APP_PERMISSIONS + {"perm list", command_perm_list, 0}, + {"perm grant", command_perm_grant, 2}, + {"perm revoke", command_perm_revoke, 2}, +#endif #endif // End of automation commands // ==================================================================================== diff --git a/src/fw/services/app_permissions/CMakeLists.txt b/src/fw/services/app_permissions/CMakeLists.txt index ee6da1422d..c43322d61c 100644 --- a/src/fw/services/app_permissions/CMakeLists.txt +++ b/src/fw/services/app_permissions/CMakeLists.txt @@ -2,4 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 pbl_library() -pbl_library_sources(app_permissions.c) +pbl_library_sources( + app_permissions.c + app_permissions_commands.c +) diff --git a/src/fw/services/app_permissions/app_permissions_commands.c b/src/fw/services/app_permissions/app_permissions_commands.c new file mode 100644 index 0000000000..45b849ce09 --- /dev/null +++ b/src/fw/services/app_permissions/app_permissions_commands.c @@ -0,0 +1,73 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "console/prompt.h" +#include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/blob_db/app_permissions_db.h" +#include "pbl/services/comm_session/session.h" +#include "process_management/app_install_manager.h" +#include "syscall/syscall.h" + +#include +#include +#include + +static const char *s_permission_names[AppPermissionCount] = { + [AppPermission_Microphone] = "mic", +}; + +static bool prv_parse_permission(const char *name, AppPermission *permission_out) { + for (AppPermission p = 0; p < AppPermissionCount; p++) { + if (strcmp(name, s_permission_names[p]) == 0) { + *permission_out = p; + return true; + } + } + return false; +} + +static bool prv_list_record(const Uuid *uuid, const AppPermissionsDBEntry *entry, void *context) { + char uuid_str[UUID_STRING_BUFFER_LENGTH]; + uuid_to_string(uuid, uuid_str); + char buffer[96]; + prompt_send_response_fmt( + buffer, sizeof(buffer), "%s id=%" PRId32 " granted=0x%" PRIx32 " declared=0x%" PRIx32, + uuid_str, app_install_get_id_for_uuid(uuid), entry->granted_mask, entry->declared_mask); + return true; +} + +void command_perm_list(void) { + char buffer[64]; + prompt_send_response_fmt( + buffer, sizeof(buffer), "phone support: %s", + sys_system_pp_has_capability(CommSessionAppPermissionsSupport) ? "yes" : "no"); + app_permissions_db_each(prv_list_record, NULL); + prompt_send_response("OK"); +} + +static void prv_set(const char *id_str, const char *perm_str, bool granted) { + const AppInstallId id = atoi(id_str); + Uuid uuid; + if ((id == INSTALL_ID_INVALID) || !app_install_get_uuid_for_install_id(id, &uuid)) { + prompt_send_response("No app with id"); + return; + } + AppPermission permission; + if (!prv_parse_permission(perm_str, &permission)) { + prompt_send_response("Unknown permission (try: mic)"); + return; + } + if (app_permissions_set_granted(&uuid, permission, granted) != S_SUCCESS) { + prompt_send_response("Failed"); + return; + } + prompt_send_response("OK"); +} + +void command_perm_grant(const char *id_str, const char *perm_str) { + prv_set(id_str, perm_str, true); +} + +void command_perm_revoke(const char *id_str, const char *perm_str) { + prv_set(id_str, perm_str, false); +} From 25a7ae564fe26249bcd064eb88078f20fce80507 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:41:01 +0200 Subject: [PATCH 06/15] services: add mic_manager arbiter The mic driver has a single client and no arbitration. Introduce a small kernel arbiter so dictation and the upcoming app capture service can share it: dictation always wins and preempts an app (which is told through a callback before the mic restarts), while an app is refused while dictation runs. Route the dictation service through it. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- include/pbl/services/mic_manager.h | 39 +++++ src/fw/services/CMakeLists.txt | 1 + src/fw/services/Kconfig | 1 + src/fw/services/mic_manager/CMakeLists.txt | 5 + src/fw/services/mic_manager/Kconfig | 17 ++ src/fw/services/mic_manager/mic_manager.c | 108 ++++++++++++ src/fw/services/services_normal/service.c | 4 + src/fw/services/voice/Kconfig | 1 + src/fw/services/voice/voice.c | 8 +- tests/fw/services/CMakeLists.txt | 6 + tests/fw/services/test_mic_manager.c | 194 +++++++++++++++++++++ 11 files changed, 381 insertions(+), 3 deletions(-) create mode 100644 include/pbl/services/mic_manager.h create mode 100644 src/fw/services/mic_manager/CMakeLists.txt create mode 100644 src/fw/services/mic_manager/Kconfig create mode 100644 src/fw/services/mic_manager/mic_manager.c create mode 100644 tests/fw/services/test_mic_manager.c diff --git a/include/pbl/services/mic_manager.h b/include/pbl/services/mic_manager.h new file mode 100644 index 0000000000..ad6a4532ef --- /dev/null +++ b/include/pbl/services/mic_manager.h @@ -0,0 +1,39 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include + +#include +#include +#include + +//! Arbitrates the single microphone between kernel clients. +//! +//! Dictation (the system voice UI) has priority over app capture: acquiring for dictation while an +//! app owns the mic preempts the app, which is told through its preempted callback before the mic +//! is restarted. An app cannot acquire the mic while dictation owns it. + +typedef enum MicClient { + MicClientNone = 0, + MicClientVoiceDictation, + MicClientAppCapture, +} MicClient; + +//! Called (without the manager lock held) when the client's mic is taken away. The client must +//! not call mic_manager_release() for the lost session; it is already released. +typedef void (*MicManagerPreemptedCb)(void *context); + +void mic_manager_init(void); + +//! Starts the microphone for a client. See \ref mic_start for the buffer semantics. +//! @return true if the client now owns the running microphone +bool mic_manager_acquire(MicClient client, MicDataHandlerCB handler, void *context, int16_t *buffer, + size_t buffer_len, MicManagerPreemptedCb on_preempted, + void *preempt_context); + +//! Stops the microphone if the client owns it. No-op otherwise. +void mic_manager_release(MicClient client); + +MicClient mic_manager_get_owner(void); diff --git a/src/fw/services/CMakeLists.txt b/src/fw/services/CMakeLists.txt index 6972cf833f..06a37e69d1 100644 --- a/src/fw/services/CMakeLists.txt +++ b/src/fw/services/CMakeLists.txt @@ -10,6 +10,7 @@ pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_CACHE app_cache) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_FETCH_ENDPOINT app_fetch_endpoint) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_GLANCES app_glances) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_PERMISSIONS app_permissions) +pbl_add_subdirectory_ifdef(CONFIG_SERVICE_MIC_MANAGER mic_manager) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_INBOX_SERVICE app_inbox_service) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_MESSAGE app_message) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_ORDER_ENDPOINT app_order_endpoint) diff --git a/src/fw/services/Kconfig b/src/fw/services/Kconfig index b8f1ebf641..43fd846e01 100644 --- a/src/fw/services/Kconfig +++ b/src/fw/services/Kconfig @@ -12,6 +12,7 @@ rsource "app_cache/Kconfig" rsource "app_fetch_endpoint/Kconfig" rsource "app_glances/Kconfig" rsource "app_permissions/Kconfig" +rsource "mic_manager/Kconfig" rsource "app_inbox_service/Kconfig" rsource "app_message/Kconfig" rsource "app_order_endpoint/Kconfig" diff --git a/src/fw/services/mic_manager/CMakeLists.txt b/src/fw/services/mic_manager/CMakeLists.txt new file mode 100644 index 0000000000..25f95cca79 --- /dev/null +++ b/src/fw/services/mic_manager/CMakeLists.txt @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +pbl_library() +pbl_library_sources(mic_manager.c) diff --git a/src/fw/services/mic_manager/Kconfig b/src/fw/services/mic_manager/Kconfig new file mode 100644 index 0000000000..fcb0d41c93 --- /dev/null +++ b/src/fw/services/mic_manager/Kconfig @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +config SERVICE_MIC_MANAGER + bool "Microphone manager" + depends on MIC + default y + help + Arbitrates the microphone between dictation and app capture. + +if SERVICE_MIC_MANAGER + +module = SERVICE_MIC_MANAGER +module-str = Mic manager +source "subsys/logging/Kconfig.template.log_level" + +endif diff --git a/src/fw/services/mic_manager/mic_manager.c b/src/fw/services/mic_manager/mic_manager.c new file mode 100644 index 0000000000..d7ff1c693f --- /dev/null +++ b/src/fw/services/mic_manager/mic_manager.c @@ -0,0 +1,108 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "pbl/services/mic_manager.h" + +#include "board/board.h" +#include "pbl/kernel/mutex.h" +#include + +PBL_LOG_MODULE_DEFINE(service_mic_manager, CONFIG_SERVICE_MIC_MANAGER_LOG_LEVEL); + +static PBL_MUTEX_DEFINE(s_lock); + +typedef struct { + MicClient owner; + MicManagerPreemptedCb on_preempted; + void *preempt_context; + //! Set while dictation is taking the mic away from an app, so the app cannot slip back in + //! between the preempt callback and the restart. + bool preempting; +} MicManagerState; + +static MicManagerState s_state; + +void mic_manager_init(void) { + s_state = (MicManagerState){}; +} + +static bool prv_start_locked(MicClient client, MicDataHandlerCB handler, void *context, + int16_t *buffer, size_t buffer_len, MicManagerPreemptedCb on_preempted, + void *preempt_context) { + if (!mic_start(MIC, handler, context, buffer, buffer_len)) { + PBL_LOG_WRN("Mic failed to start for client %u", client); + return false; + } + s_state.owner = client; + s_state.on_preempted = on_preempted; + s_state.preempt_context = preempt_context; + PBL_LOG_DBG("Mic acquired by client %u", client); + return true; +} + +bool mic_manager_acquire(MicClient client, MicDataHandlerCB handler, void *context, int16_t *buffer, + size_t buffer_len, MicManagerPreemptedCb on_preempted, + void *preempt_context) { + if ((client == MicClientNone) || !handler || !buffer || (buffer_len == 0)) { + return false; + } + + pbl_mutex_lock(&s_lock, PBL_FOREVER); + + if (s_state.preempting || (s_state.owner == client)) { + pbl_mutex_unlock(&s_lock); + return false; + } + + if (s_state.owner == MicClientNone) { + const bool rv = prv_start_locked(client, handler, context, buffer, buffer_len, on_preempted, + preempt_context); + pbl_mutex_unlock(&s_lock); + return rv; + } + + if (client != MicClientVoiceDictation) { + // Apps never preempt anyone + PBL_LOG_DBG("Mic busy (owner %u), refusing client %u", s_state.owner, client); + pbl_mutex_unlock(&s_lock); + return false; + } + + // Dictation takes the mic away from the app + PBL_LOG_DBG("Dictation preempting mic owner %u", s_state.owner); + const MicManagerPreemptedCb preempted_cb = s_state.on_preempted; + void *preempted_ctx = s_state.preempt_context; + mic_stop(MIC); + s_state.owner = MicClientNone; + s_state.on_preempted = NULL; + s_state.preempt_context = NULL; + s_state.preempting = true; + pbl_mutex_unlock(&s_lock); + + if (preempted_cb) { + preempted_cb(preempted_ctx); + } + + pbl_mutex_lock(&s_lock, PBL_FOREVER); + s_state.preempting = false; + const bool rv = + prv_start_locked(client, handler, context, buffer, buffer_len, on_preempted, preempt_context); + pbl_mutex_unlock(&s_lock); + return rv; +} + +void mic_manager_release(MicClient client) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if ((client != MicClientNone) && (s_state.owner == client)) { + mic_stop(MIC); + s_state.owner = MicClientNone; + s_state.on_preempted = NULL; + s_state.preempt_context = NULL; + PBL_LOG_DBG("Mic released by client %u", client); + } + pbl_mutex_unlock(&s_lock); +} + +MicClient mic_manager_get_owner(void) { + return s_state.owner; +} diff --git a/src/fw/services/services_normal/service.c b/src/fw/services/services_normal/service.c index 2d6bf84f56..1337512bc9 100644 --- a/src/fw/services/services_normal/service.c +++ b/src/fw/services/services_normal/service.c @@ -14,6 +14,7 @@ #include "pbl/services/app_cache.h" #include "pbl/services/app_glances/app_glance_service.h" #include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/mic_manager.h" #include "pbl/services/blob_db/api.h" #include "pbl/services/blob_db/endpoint_private.h" #include "pbl/services/data_logging/data_logging_service.h" @@ -121,6 +122,9 @@ void services_normal_init(void) { speaker_service_init(); +#ifdef CONFIG_SERVICE_MIC_MANAGER + mic_manager_init(); +#endif #ifdef CONFIG_MIC voice_init(); #endif diff --git a/src/fw/services/voice/Kconfig b/src/fw/services/voice/Kconfig index 25957bed43..e45b48289c 100644 --- a/src/fw/services/voice/Kconfig +++ b/src/fw/services/voice/Kconfig @@ -5,6 +5,7 @@ config SERVICE_VOICE bool "Voice" default y if MIC select SPEEX + select SERVICE_MIC_MANAGER help Voice (dictation) service. Requires a microphone driver (CONFIG_MIC). diff --git a/src/fw/services/voice/voice.c b/src/fw/services/voice/voice.c index 427601b6bc..44887ffcd6 100644 --- a/src/fw/services/voice/voice.c +++ b/src/fw/services/voice/voice.c @@ -11,6 +11,7 @@ #include "pbl/kernel/mutex.h" #include "process_management/app_manager.h" #include "pbl/services/comm_session/session.h" +#include "pbl/services/mic_manager.h" #include "pbl/services/new_timer/new_timer.h" #include "pbl/services/audio_endpoint.h" #include "pbl/services/voice/transcription.h" @@ -118,7 +119,7 @@ static void prv_stop_recording(void) { // This prevents new frames from being added while the endpoint shuts down audio_endpoint_stop_transfer(s_session_id); - mic_stop(MIC); + mic_manager_release(MicClientVoiceDictation); prv_teardown_session(); @@ -127,7 +128,7 @@ static void prv_stop_recording(void) { static void prv_cancel_recording(void) { PBL_LOG_DBG("prv_cancel_recording called - cancelling mic and audio endpoint transfer"); - mic_stop(MIC); + mic_manager_release(MicClientVoiceDictation); audio_endpoint_cancel_transfer(s_session_id); prv_teardown_session(); @@ -196,7 +197,8 @@ static bool prv_start_recording(void) { if (frame_buffer && frame_size_samples > 0) { PBL_LOG_DBG("Starting microphone with frame buffer"); - if (!mic_start(MIC, &prv_audio_data_handler, NULL, frame_buffer, frame_size_samples)) { + if (!mic_manager_acquire(MicClientVoiceDictation, &prv_audio_data_handler, NULL, frame_buffer, + frame_size_samples, NULL, NULL)) { PBL_LOG_ERR("Failed to start microphone for voice session"); return false; } diff --git a/tests/fw/services/CMakeLists.txt b/tests/fw/services/CMakeLists.txt index 0147219371..92638b80f9 100644 --- a/tests/fw/services/CMakeLists.txt +++ b/tests/fw/services/CMakeLists.txt @@ -305,6 +305,12 @@ pbl_clar_test(test_app_cache OVERRIDES dummy_board ) +pbl_clar_test(test_mic_manager + SOURCES + src/fw/services/mic_manager/mic_manager.c + OVERRIDES dummy_board +) + pbl_clar_test(test_app_permissions SOURCES src/fw/services/app_permissions/app_permissions.c diff --git a/tests/fw/services/test_mic_manager.c b/tests/fw/services/test_mic_manager.c new file mode 100644 index 0000000000..dba26eab22 --- /dev/null +++ b/tests/fw/services/test_mic_manager.c @@ -0,0 +1,194 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "clar.h" + +#include "pbl/services/mic_manager.h" + +#include + +// Fakes +//////////////////////////////////////////////////////////////// +#include "fake_mutex.h" + +// Stubs +//////////////////////////////////////////////////////////////// +#include "stubs_logging.h" +#include "stubs_passert.h" + +// Fake mic driver +//////////////////////////////////////////////////////////////// + +MicDevice *const MIC = NULL; + +static bool s_mic_running; +static int s_start_count; +static int s_stop_count; +static MicDataHandlerCB s_handler; +static int16_t *s_buffer; + +bool mic_start(MicDevice *this, MicDataHandlerCB data_handler, void *context, int16_t *audio_buffer, + size_t audio_buffer_len) { + if (s_mic_running) { + return false; + } + s_mic_running = true; + s_start_count++; + s_handler = data_handler; + s_buffer = audio_buffer; + return true; +} + +void mic_stop(MicDevice *this) { + s_mic_running = false; + s_stop_count++; + s_handler = NULL; + s_buffer = NULL; +} + +// Clients +//////////////////////////////////////////////////////////////// + +static int16_t s_dictation_buffer[320]; +static int16_t s_app_buffer[160]; + +static void prv_dictation_handler(int16_t *samples, size_t sample_count, void *context) { +} + +static void prv_app_handler(int16_t *samples, size_t sample_count, void *context) { +} + +static int s_preempted_count; +static MicClient s_owner_seen_in_preempt; +static bool s_reacquire_in_preempt; +static bool s_reacquire_result; + +static void prv_app_preempted(void *context) { + s_preempted_count++; + s_owner_seen_in_preempt = mic_manager_get_owner(); + // A client that misbehaves and releases anyway must not affect the new owner + mic_manager_release(MicClientAppCapture); + if (s_reacquire_in_preempt) { + s_reacquire_result = mic_manager_acquire(MicClientAppCapture, prv_app_handler, NULL, + s_app_buffer, 160, prv_app_preempted, NULL); + } +} + +static bool prv_acquire_app(void) { + return mic_manager_acquire(MicClientAppCapture, prv_app_handler, NULL, s_app_buffer, 160, + prv_app_preempted, NULL); +} + +static bool prv_acquire_dictation(void) { + return mic_manager_acquire(MicClientVoiceDictation, prv_dictation_handler, NULL, + s_dictation_buffer, 320, NULL, NULL); +} + +// Setup +//////////////////////////////////////////////////////////////// + +void test_mic_manager__initialize(void) { + fake_mutex_reset(false); + s_mic_running = false; + s_start_count = 0; + s_stop_count = 0; + s_handler = NULL; + s_buffer = NULL; + s_preempted_count = 0; + s_owner_seen_in_preempt = MicClientNone; + s_reacquire_in_preempt = false; + s_reacquire_result = true; + mic_manager_init(); +} + +void test_mic_manager__cleanup(void) { + fake_mutex_assert_all_unlocked(); +} + +// Tests +//////////////////////////////////////////////////////////////// + +void test_mic_manager__acquire_release(void) { + cl_assert_equal_i(MicClientNone, mic_manager_get_owner()); + + cl_assert(prv_acquire_app()); + cl_assert_equal_i(MicClientAppCapture, mic_manager_get_owner()); + cl_assert_equal_i(1, s_start_count); + cl_assert(s_handler == prv_app_handler); + cl_assert(s_buffer == s_app_buffer); + + // Same client twice is refused + cl_assert(!prv_acquire_app()); + cl_assert_equal_i(1, s_start_count); + + mic_manager_release(MicClientAppCapture); + cl_assert_equal_i(MicClientNone, mic_manager_get_owner()); + cl_assert_equal_i(1, s_stop_count); + cl_assert(!s_mic_running); + + // Release when not owning is a no-op + mic_manager_release(MicClientAppCapture); + mic_manager_release(MicClientNone); + cl_assert_equal_i(1, s_stop_count); +} + +void test_mic_manager__invalid_args(void) { + cl_assert( + !mic_manager_acquire(MicClientNone, prv_app_handler, NULL, s_app_buffer, 160, NULL, NULL)); + cl_assert(!mic_manager_acquire(MicClientAppCapture, NULL, NULL, s_app_buffer, 160, NULL, NULL)); + cl_assert( + !mic_manager_acquire(MicClientAppCapture, prv_app_handler, NULL, NULL, 160, NULL, NULL)); + cl_assert(!mic_manager_acquire(MicClientAppCapture, prv_app_handler, NULL, s_app_buffer, 0, NULL, + NULL)); + cl_assert_equal_i(0, s_start_count); +} + +void test_mic_manager__app_refused_while_dictation_owns(void) { + cl_assert(prv_acquire_dictation()); + cl_assert(!prv_acquire_app()); + cl_assert_equal_i(MicClientVoiceDictation, mic_manager_get_owner()); + cl_assert_equal_i(1, s_start_count); + + // App release while dictation owns changes nothing + mic_manager_release(MicClientAppCapture); + cl_assert(s_mic_running); + cl_assert_equal_i(MicClientVoiceDictation, mic_manager_get_owner()); + + mic_manager_release(MicClientVoiceDictation); + cl_assert_equal_i(MicClientNone, mic_manager_get_owner()); +} + +void test_mic_manager__dictation_preempts_app(void) { + cl_assert(prv_acquire_app()); + + cl_assert(prv_acquire_dictation()); + cl_assert_equal_i(1, s_preempted_count); + // The app was already released when told, so its own release was a no-op + cl_assert_equal_i(MicClientNone, s_owner_seen_in_preempt); + cl_assert_equal_i(MicClientVoiceDictation, mic_manager_get_owner()); + cl_assert_equal_i(2, s_start_count); + cl_assert_equal_i(1, s_stop_count); + cl_assert(s_handler == prv_dictation_handler); + cl_assert(s_buffer == s_dictation_buffer); + + mic_manager_release(MicClientVoiceDictation); + cl_assert(!s_mic_running); +} + +void test_mic_manager__app_cannot_reacquire_during_preempt(void) { + cl_assert(prv_acquire_app()); + s_reacquire_in_preempt = true; + + cl_assert(prv_acquire_dictation()); + cl_assert_equal_i(1, s_preempted_count); + cl_assert(!s_reacquire_result); + cl_assert_equal_i(MicClientVoiceDictation, mic_manager_get_owner()); + cl_assert(s_handler == prv_dictation_handler); +} + +void test_mic_manager__driver_start_failure(void) { + // Simulate the driver being busy outside our control + s_mic_running = true; + cl_assert(!prv_acquire_app()); + cl_assert_equal_i(MicClientNone, mic_manager_get_owner()); +} From e90853309771357d862003775ac3886cc38fbb31 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:44:58 +0200 Subject: [PATCH 07/15] services: add mic_capture_service Kernel side of live PCM capture for apps. Serves the app task only, requires focus and the microphone permission, acquires the mic through mic_manager and buffers samples in a 320 ms kernel ring the app drains through syscalls, with coalesced PEBBLE_MIC_CAPTURE_EVENT data events. Capture stops, with a reason reported to the app, when the app loses focus to a modal window, when the grant is revoked, or when dictation preempts the mic. App exit tears it down silently. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- .../mic_capture/mic_capture_service.h | 69 ++++ src/fw/kernel/events.h | 14 + src/fw/kernel/ui/modals/modal_manager.c | 9 + src/fw/process_management/app_manager.c | 6 + src/fw/services/CMakeLists.txt | 1 + src/fw/services/Kconfig | 1 + src/fw/services/mic_capture/CMakeLists.txt | 5 + src/fw/services/mic_capture/Kconfig | 17 + .../mic_capture/mic_capture_service.c | 239 +++++++++++++ src/fw/services/services_normal/service.c | 4 + tests/fw/services/CMakeLists.txt | 9 + tests/fw/services/test_mic_capture_service.c | 317 ++++++++++++++++++ 12 files changed, 691 insertions(+) create mode 100644 include/pbl/services/mic_capture/mic_capture_service.h create mode 100644 src/fw/services/mic_capture/CMakeLists.txt create mode 100644 src/fw/services/mic_capture/Kconfig create mode 100644 src/fw/services/mic_capture/mic_capture_service.c create mode 100644 tests/fw/services/test_mic_capture_service.c diff --git a/include/pbl/services/mic_capture/mic_capture_service.h b/include/pbl/services/mic_capture/mic_capture_service.h new file mode 100644 index 0000000000..efe90997e3 --- /dev/null +++ b/include/pbl/services/mic_capture/mic_capture_service.h @@ -0,0 +1,69 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "kernel/pebble_tasks.h" + +#include +#include + +//! Live PCM capture for the foreground app. +//! +//! The service owns the mic (through mic_manager), a kernel ring buffer the app drains through +//! syscalls, and the OS "listening" banner. It only ever serves the app task, only while the app +//! is in focus and holds the microphone permission. Capture stops on any focus loss, when the +//! grant is revoked, when dictation preempts the mic, or when the app goes away. +//! +//! Data flow: the mic driver hands chunks of samples_per_update samples on KernelBG; they are +//! appended to the ring buffer (dropping the newest chunk when full) and a coalesced +//! PEBBLE_MIC_CAPTURE_EVENT tells the app to read. + +typedef enum MicCaptureStartResult { + MicCaptureStartOk = 0, + MicCaptureStartErrNotDeclared, + MicCaptureStartErrDenied, + MicCaptureStartErrBusy, + MicCaptureStartErrNotForeground, + MicCaptureStartErrInvalidArgs, + MicCaptureStartErrNoMemory, +} MicCaptureStartResult; + +typedef enum MicCaptureStopReason { + MicCaptureStopReasonStopped = 0, + MicCaptureStopReasonFocusLost, + MicCaptureStopReasonPreempted, + MicCaptureStopReasonPermissionRevoked, + MicCaptureStopReasonAppExit, + MicCaptureStopReasonError, +} MicCaptureStopReason; + +#define MIC_CAPTURE_SAMPLE_RATE (16000) +#define MIC_CAPTURE_MIN_SAMPLES_PER_UPDATE (80) // 5 ms +#define MIC_CAPTURE_MAX_SAMPLES_PER_UPDATE (1600) // 100 ms +#define MIC_CAPTURE_RING_SAMPLES (5120) // 320 ms + +void mic_capture_service_init(void); + +//! Starts capture for the given task (must be the app task, in focus, with the permission). +MicCaptureStartResult mic_capture_service_start(PebbleTask owner, uint16_t samples_per_update); + +//! Stops capture at the owner's request. No stop event is sent. +void mic_capture_service_stop(PebbleTask owner); + +//! Stops capture because the task is going away. No stop event is sent. Safe to call when idle. +void mic_capture_service_stop_for_task(PebbleTask task); + +//! Copies up to max_samples out of the ring buffer and consumes them. +//! @return number of samples copied +uint32_t mic_capture_service_read(PebbleTask owner, int16_t *out, uint32_t max_samples); + +uint32_t mic_capture_service_get_available(void); + +bool mic_capture_service_is_active(void); + +//! The app lost focus to a modal window: capture stops with MicCaptureStopReasonFocusLost. +void mic_capture_service_handle_app_focus_lost(void); + +//! The running app's grants changed: capture stops if the mic is no longer granted. +void mic_capture_service_handle_permission_changed(void); diff --git a/src/fw/kernel/events.h b/src/fw/kernel/events.h index ddd19ea440..f0fb5b34a5 100644 --- a/src/fw/kernel/events.h +++ b/src/fw/kernel/events.h @@ -129,6 +129,7 @@ typedef enum { PEBBLE_SPEAKER_EVENT, PEBBLE_BACKLIGHT_EVENT, PEBBLE_APP_PERMISSION_EVENT, + PEBBLE_MIC_CAPTURE_EVENT, PEBBLE_NUM_EVENTS } PebbleEventType; @@ -504,6 +505,18 @@ typedef struct PACKED { // 2 bytes uint8_t state; //!< AppPermissionState } PebbleAppPermissionEvent; +typedef enum { + MicCaptureEventData = 0, + MicCaptureEventStopped = 1, +} MicCaptureEventType; + +typedef struct PACKED { // 5 bytes + uint8_t type; //!< MicCaptureEventType + uint8_t stop_reason; //!< MicCaptureStopReason (Stopped events only) + bool overrun; //!< Samples were dropped since the last data event + uint16_t num_samples; //!< Samples available when the event was posted +} PebbleMicCaptureEvent; + typedef enum { VoiceEventTypeSessionSetup, VoiceEventTypeSessionResult, @@ -815,6 +828,7 @@ typedef struct PACKED { PebbleSpeakerEvent speaker; PebbleBacklightEvent backlight; PebbleAppPermissionEvent app_permission; + PebbleMicCaptureEvent mic_capture; }; PebbleTaskBitset task_mask; // 1 == filter out, 0 == leave in // NOTE: we put this 8 bit field at the end so that we can pack this structure and still keep the diff --git a/src/fw/kernel/ui/modals/modal_manager.c b/src/fw/kernel/ui/modals/modal_manager.c index ab4ee3dc11..eed6445176 100644 --- a/src/fw/kernel/ui/modals/modal_manager.c +++ b/src/fw/kernel/ui/modals/modal_manager.c @@ -3,6 +3,10 @@ #include "modal_manager.h" +#ifdef CONFIG_SERVICE_MIC_CAPTURE +#include "pbl/services/mic_capture/mic_capture_service.h" +#endif + #include "applib/ui/app_window_click_glue.h" #include "applib/ui/click_internal.h" #include "applib/ui/recognizer/recognizer_list.h" @@ -327,6 +331,11 @@ static void prv_handle_app_to_modal_transition_focus(void) { // Let the underlying window know it has lost focus if this is the first modal // window to show up. prv_send_will_focus_event(false /* in_focus */); + +#ifdef CONFIG_SERVICE_MIC_CAPTURE + // The OS mic banner is no longer guaranteed visible, so live capture must end. + mic_capture_service_handle_app_focus_lost(); +#endif } static void prv_handle_modal_to_app_transition_focus(void) { diff --git a/src/fw/process_management/app_manager.c b/src/fw/process_management/app_manager.c index bb051de229..53160b2cb7 100644 --- a/src/fw/process_management/app_manager.c +++ b/src/fw/process_management/app_manager.c @@ -35,6 +35,9 @@ #include "pbl/services/vibe_pattern.h" #ifndef CONFIG_RECOVERY_FW #include "pbl/services/speaker/speaker_service.h" +#ifdef CONFIG_SERVICE_MIC_CAPTURE +#include "pbl/services/mic_capture/mic_capture_service.h" +#endif #endif #include "shell/normal/app_idle_timeout.h" #include "shell/normal/watchface.h" @@ -411,6 +414,9 @@ static void prv_app_cleanup(void) { vibe_pattern_clear_for_owner(VibePatternOwner_App); #ifndef CONFIG_RECOVERY_FW speaker_service_stop_for_task(PebbleTask_App); +#endif +#ifdef CONFIG_SERVICE_MIC_CAPTURE + mic_capture_service_stop_for_task(PebbleTask_App); #endif ble_app_cleanup(); diff --git a/src/fw/services/CMakeLists.txt b/src/fw/services/CMakeLists.txt index 06a37e69d1..ba614554f9 100644 --- a/src/fw/services/CMakeLists.txt +++ b/src/fw/services/CMakeLists.txt @@ -11,6 +11,7 @@ pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_FETCH_ENDPOINT app_fetch_endpoint) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_GLANCES app_glances) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_PERMISSIONS app_permissions) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_MIC_MANAGER mic_manager) +pbl_add_subdirectory_ifdef(CONFIG_SERVICE_MIC_CAPTURE mic_capture) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_INBOX_SERVICE app_inbox_service) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_MESSAGE app_message) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_ORDER_ENDPOINT app_order_endpoint) diff --git a/src/fw/services/Kconfig b/src/fw/services/Kconfig index 43fd846e01..a859713e8b 100644 --- a/src/fw/services/Kconfig +++ b/src/fw/services/Kconfig @@ -13,6 +13,7 @@ rsource "app_fetch_endpoint/Kconfig" rsource "app_glances/Kconfig" rsource "app_permissions/Kconfig" rsource "mic_manager/Kconfig" +rsource "mic_capture/Kconfig" rsource "app_inbox_service/Kconfig" rsource "app_message/Kconfig" rsource "app_order_endpoint/Kconfig" diff --git a/src/fw/services/mic_capture/CMakeLists.txt b/src/fw/services/mic_capture/CMakeLists.txt new file mode 100644 index 0000000000..83153195f4 --- /dev/null +++ b/src/fw/services/mic_capture/CMakeLists.txt @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +pbl_library() +pbl_library_sources(mic_capture_service.c) diff --git a/src/fw/services/mic_capture/Kconfig b/src/fw/services/mic_capture/Kconfig new file mode 100644 index 0000000000..9b36d81b93 --- /dev/null +++ b/src/fw/services/mic_capture/Kconfig @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +config SERVICE_MIC_CAPTURE + bool "Microphone capture for apps" + depends on SERVICE_MIC_MANAGER && SERVICE_APP_PERMISSIONS + default y + help + Live PCM capture service backing the app microphone API. + +if SERVICE_MIC_CAPTURE + +module = SERVICE_MIC_CAPTURE +module-str = Mic capture +source "subsys/logging/Kconfig.template.log_level" + +endif diff --git a/src/fw/services/mic_capture/mic_capture_service.c b/src/fw/services/mic_capture/mic_capture_service.c new file mode 100644 index 0000000000..ebefe0d519 --- /dev/null +++ b/src/fw/services/mic_capture/mic_capture_service.c @@ -0,0 +1,239 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "pbl/services/mic_capture/mic_capture_service.h" + +#include "kernel/events.h" +#include "kernel/pbl_malloc.h" +#include "kernel/ui/modals/modal_manager.h" +#include "pbl/kernel/mutex.h" +#include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/mic_manager.h" +#include "pbl/util/circular_buffer.h" +#include + +#include + +PBL_LOG_MODULE_DEFINE(service_mic_capture, CONFIG_SERVICE_MIC_CAPTURE_LOG_LEVEL); + +#define RING_BYTES (MIC_CAPTURE_RING_SAMPLES * sizeof(int16_t)) + +typedef struct { + bool active; + PebbleTask owner; + uint16_t samples_per_update; + CircularBuffer ring; + uint8_t *ring_storage; + int16_t *chunk; + bool overrun; + bool data_event_pending; +} MicCaptureState; + +static PBL_MUTEX_DEFINE(s_lock); +static MicCaptureState s_state; + +void mic_capture_service_init(void) { + s_state = (MicCaptureState){}; +} + +static void prv_post_event(uint8_t type, MicCaptureStopReason reason, bool overrun, + uint16_t num_samples) { + PebbleEvent e = { + .type = PEBBLE_MIC_CAPTURE_EVENT, + .mic_capture = { + .type = type, + .stop_reason = (uint8_t)reason, + .overrun = overrun, + .num_samples = num_samples, + }, + }; + event_put(&e); +} + +//! Expects s_lock held. Frees resources and returns the buffers to free. +static void prv_teardown_locked(void) { + s_state.active = false; + s_state.owner = PebbleTask_Unknown; + s_state.data_event_pending = false; + kernel_free(s_state.ring_storage); + kernel_free(s_state.chunk); + s_state.ring_storage = NULL; + s_state.chunk = NULL; +} + +//! Stops capture for a system-originated reason and tells the app. `release_mic` is false when +//! the mic manager already took the mic away (preemption). +static void prv_stop_with_reason(MicCaptureStopReason reason, bool release_mic) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (!s_state.active) { + pbl_mutex_unlock(&s_lock); + return; + } + PBL_LOG_DBG("Capture stopped, reason %u", reason); + prv_teardown_locked(); + pbl_mutex_unlock(&s_lock); + + if (release_mic) { + mic_manager_release(MicClientAppCapture); + } + prv_post_event(MicCaptureEventStopped, reason, false, 0); +} + +// Runs on KernelBG, driven by the mic driver. +static void prv_mic_data_handler(int16_t *samples, size_t sample_count, void *context) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (!s_state.active) { + pbl_mutex_unlock(&s_lock); + return; + } + const uint16_t bytes = sample_count * sizeof(int16_t); + if (!circular_buffer_write(&s_state.ring, samples, bytes)) { + // Full: drop the newest chunk, the app is not keeping up + s_state.overrun = true; + } + bool post = false; + bool overrun = false; + uint16_t available = 0; + if (!s_state.data_event_pending) { + s_state.data_event_pending = true; + post = true; + overrun = s_state.overrun; + s_state.overrun = false; + available = circular_buffer_get_read_space_remaining(&s_state.ring) / sizeof(int16_t); + } + pbl_mutex_unlock(&s_lock); + + if (post) { + prv_post_event(MicCaptureEventData, MicCaptureStopReasonStopped, overrun, available); + } +} + +static void prv_preempted(void *context) { + prv_stop_with_reason(MicCaptureStopReasonPreempted, false /* mic already gone */); +} + +static bool prv_app_is_in_focus(void) { + return !modal_manager_get_enabled() || (modal_manager_get_properties() & ModalProperty_Unfocused); +} + +MicCaptureStartResult mic_capture_service_start(PebbleTask owner, uint16_t samples_per_update) { + if (owner != PebbleTask_App) { + return MicCaptureStartErrNotForeground; + } + if ((samples_per_update < MIC_CAPTURE_MIN_SAMPLES_PER_UPDATE) || + (samples_per_update > MIC_CAPTURE_MAX_SAMPLES_PER_UPDATE)) { + return MicCaptureStartErrInvalidArgs; + } + if (!prv_app_is_in_focus()) { + return MicCaptureStartErrNotForeground; + } + switch (app_permissions_get_state_for_current_app(AppPermission_Microphone)) { + case AppPermissionStateNotDeclared: + return MicCaptureStartErrNotDeclared; + case AppPermissionStateDenied: + return MicCaptureStartErrDenied; + case AppPermissionStateGranted: + break; + } + + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (s_state.active) { + pbl_mutex_unlock(&s_lock); + return MicCaptureStartErrBusy; + } + + uint8_t *ring_storage = kernel_malloc(RING_BYTES); + int16_t *chunk = kernel_malloc(samples_per_update * sizeof(int16_t)); + if (!ring_storage || !chunk) { + kernel_free(ring_storage); + kernel_free(chunk); + pbl_mutex_unlock(&s_lock); + return MicCaptureStartErrNoMemory; + } + + s_state = (MicCaptureState){ + .active = true, + .owner = owner, + .samples_per_update = samples_per_update, + .ring_storage = ring_storage, + .chunk = chunk, + }; + circular_buffer_init(&s_state.ring, ring_storage, RING_BYTES); + pbl_mutex_unlock(&s_lock); + + if (!mic_manager_acquire(MicClientAppCapture, prv_mic_data_handler, NULL, chunk, + samples_per_update, prv_preempted, NULL)) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + prv_teardown_locked(); + pbl_mutex_unlock(&s_lock); + return MicCaptureStartErrBusy; + } + + PBL_LOG_DBG("Capture started, %u samples per update", samples_per_update); + return MicCaptureStartOk; +} + +static void prv_stop_silently(PebbleTask task) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (!s_state.active || (s_state.owner != task)) { + pbl_mutex_unlock(&s_lock); + return; + } + prv_teardown_locked(); + pbl_mutex_unlock(&s_lock); + mic_manager_release(MicClientAppCapture); + PBL_LOG_DBG("Capture stopped"); +} + +void mic_capture_service_stop(PebbleTask owner) { + prv_stop_silently(owner); +} + +void mic_capture_service_stop_for_task(PebbleTask task) { + prv_stop_silently(task); +} + +uint32_t mic_capture_service_read(PebbleTask owner, int16_t *out, uint32_t max_samples) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (!s_state.active || (s_state.owner != owner) || !out) { + pbl_mutex_unlock(&s_lock); + return 0; + } + s_state.data_event_pending = false; + const uint32_t available = + circular_buffer_get_read_space_remaining(&s_state.ring) / sizeof(int16_t); + const uint32_t num_samples = (max_samples < available) ? max_samples : available; + if (num_samples > 0) { + const uint16_t bytes = num_samples * sizeof(int16_t); + circular_buffer_copy(&s_state.ring, out, bytes); + circular_buffer_consume(&s_state.ring, bytes); + } + pbl_mutex_unlock(&s_lock); + return num_samples; +} + +uint32_t mic_capture_service_get_available(void) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + const uint32_t available = + s_state.active ? circular_buffer_get_read_space_remaining(&s_state.ring) / sizeof(int16_t) + : 0; + pbl_mutex_unlock(&s_lock); + return available; +} + +bool mic_capture_service_is_active(void) { + return s_state.active; +} + +void mic_capture_service_handle_app_focus_lost(void) { + prv_stop_with_reason(MicCaptureStopReasonFocusLost, true); +} + +void mic_capture_service_handle_permission_changed(void) { + if (!s_state.active) { + return; + } + if (!app_permissions_is_granted_for_current_app(AppPermission_Microphone)) { + prv_stop_with_reason(MicCaptureStopReasonPermissionRevoked, true); + } +} diff --git a/src/fw/services/services_normal/service.c b/src/fw/services/services_normal/service.c index 1337512bc9..cc3f7ddc8e 100644 --- a/src/fw/services/services_normal/service.c +++ b/src/fw/services/services_normal/service.c @@ -14,6 +14,7 @@ #include "pbl/services/app_cache.h" #include "pbl/services/app_glances/app_glance_service.h" #include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/mic_capture/mic_capture_service.h" #include "pbl/services/mic_manager.h" #include "pbl/services/blob_db/api.h" #include "pbl/services/blob_db/endpoint_private.h" @@ -125,6 +126,9 @@ void services_normal_init(void) { #ifdef CONFIG_SERVICE_MIC_MANAGER mic_manager_init(); #endif +#ifdef CONFIG_SERVICE_MIC_CAPTURE + mic_capture_service_init(); +#endif #ifdef CONFIG_MIC voice_init(); #endif diff --git a/tests/fw/services/CMakeLists.txt b/tests/fw/services/CMakeLists.txt index 92638b80f9..79dfb57440 100644 --- a/tests/fw/services/CMakeLists.txt +++ b/tests/fw/services/CMakeLists.txt @@ -311,6 +311,15 @@ pbl_clar_test(test_mic_manager OVERRIDES dummy_board ) +pbl_clar_test(test_mic_capture_service + SOURCES + src/fw/services/mic_capture/mic_capture_service.c + src/fw/services/mic_manager/mic_manager.c + lib/util/circular_buffer.c + tests/fakes/fake_events.c + OVERRIDES dummy_board +) + pbl_clar_test(test_app_permissions SOURCES src/fw/services/app_permissions/app_permissions.c diff --git a/tests/fw/services/test_mic_capture_service.c b/tests/fw/services/test_mic_capture_service.c new file mode 100644 index 0000000000..babc7dbef0 --- /dev/null +++ b/tests/fw/services/test_mic_capture_service.c @@ -0,0 +1,317 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "clar.h" + +#include "kernel/events.h" +#include "kernel/ui/modals/modal_manager.h" +#include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/mic_capture/mic_capture_service.h" +#include "pbl/services/mic_manager.h" + +#include + +// Fakes +//////////////////////////////////////////////////////////////// +#include "fake_events.h" +#include "fake_mutex.h" +#include "fake_pbl_malloc.h" + +// Stubs +//////////////////////////////////////////////////////////////// +#include "stubs_logging.h" +#include "stubs_passert.h" + +// Fake mic driver, driven by the tests +//////////////////////////////////////////////////////////////// + +MicDevice *const MIC = NULL; + +static bool s_mic_running; +static MicDataHandlerCB s_mic_handler; +static void *s_mic_context; +static int16_t *s_mic_buffer; +static size_t s_mic_buffer_len; + +bool mic_start(MicDevice *this, MicDataHandlerCB data_handler, void *context, int16_t *audio_buffer, + size_t audio_buffer_len) { + if (s_mic_running) { + return false; + } + s_mic_running = true; + s_mic_handler = data_handler; + s_mic_context = context; + s_mic_buffer = audio_buffer; + s_mic_buffer_len = audio_buffer_len; + return true; +} + +void mic_stop(MicDevice *this) { + s_mic_running = false; + s_mic_handler = NULL; +} + +//! Simulates the driver delivering one chunk of `value`-filled samples on KernelBG. +static void prv_deliver_chunk(int16_t value) { + cl_assert(s_mic_running); + for (size_t i = 0; i < s_mic_buffer_len; i++) { + s_mic_buffer[i] = value; + } + s_mic_handler(s_mic_buffer, s_mic_buffer_len, s_mic_context); +} + +// Permissions / focus +//////////////////////////////////////////////////////////////// + +static AppPermissionState s_permission_state; + +AppPermissionState app_permissions_get_state_for_current_app(AppPermission permission) { + cl_assert_equal_i(permission, AppPermission_Microphone); + return s_permission_state; +} + +bool app_permissions_is_granted_for_current_app(AppPermission permission) { + return (app_permissions_get_state_for_current_app(permission) == AppPermissionStateGranted); +} + +static bool s_modal_focused; + +bool modal_manager_get_enabled(void) { + return true; +} + +ModalProperty modal_manager_get_properties(void) { + return s_modal_focused ? ModalProperty_Exists : ModalPropertyDefault; +} + +// Dictation, used to test preemption +//////////////////////////////////////////////////////////////// + +static int16_t s_dictation_buffer[320]; + +static void prv_dictation_handler(int16_t *samples, size_t sample_count, void *context) { +} + +static bool prv_start_dictation(void) { + return mic_manager_acquire(MicClientVoiceDictation, prv_dictation_handler, NULL, + s_dictation_buffer, 320, NULL, NULL); +} + +// Helpers +//////////////////////////////////////////////////////////////// + +#define SPU (320) + +static PebbleEvent prv_last_event(void) { + return fake_event_get_last(); +} + +static void prv_assert_stopped_event(MicCaptureStopReason reason) { + PebbleEvent e = prv_last_event(); + cl_assert_equal_i(e.type, PEBBLE_MIC_CAPTURE_EVENT); + cl_assert_equal_i(e.mic_capture.type, MicCaptureEventStopped); + cl_assert_equal_i(e.mic_capture.stop_reason, reason); +} + +// Setup +//////////////////////////////////////////////////////////////// + +void test_mic_capture_service__initialize(void) { + fake_event_init(); + fake_mutex_reset(false); + fake_pbl_malloc_clear_tracking(); + s_mic_running = false; + s_mic_handler = NULL; + s_permission_state = AppPermissionStateGranted; + s_modal_focused = false; + mic_manager_init(); + mic_capture_service_init(); +} + +void test_mic_capture_service__cleanup(void) { + mic_capture_service_stop_for_task(PebbleTask_App); + mic_manager_release(MicClientVoiceDictation); + fake_mutex_assert_all_unlocked(); + fake_pbl_malloc_check_net_allocs(); +} + +// Tests +//////////////////////////////////////////////////////////////// + +void test_mic_capture_service__start_refusals(void) { + cl_assert_equal_i(MicCaptureStartErrNotForeground, + mic_capture_service_start(PebbleTask_Worker, SPU)); + cl_assert_equal_i( + MicCaptureStartErrInvalidArgs, + mic_capture_service_start(PebbleTask_App, MIC_CAPTURE_MIN_SAMPLES_PER_UPDATE - 1)); + cl_assert_equal_i( + MicCaptureStartErrInvalidArgs, + mic_capture_service_start(PebbleTask_App, MIC_CAPTURE_MAX_SAMPLES_PER_UPDATE + 1)); + + s_modal_focused = true; + cl_assert_equal_i(MicCaptureStartErrNotForeground, + mic_capture_service_start(PebbleTask_App, SPU)); + s_modal_focused = false; + + s_permission_state = AppPermissionStateNotDeclared; + cl_assert_equal_i(MicCaptureStartErrNotDeclared, mic_capture_service_start(PebbleTask_App, SPU)); + s_permission_state = AppPermissionStateDenied; + cl_assert_equal_i(MicCaptureStartErrDenied, mic_capture_service_start(PebbleTask_App, SPU)); + s_permission_state = AppPermissionStateGranted; + + // Dictation owns the mic + cl_assert(prv_start_dictation()); + cl_assert_equal_i(MicCaptureStartErrBusy, mic_capture_service_start(PebbleTask_App, SPU)); + cl_assert(!mic_capture_service_is_active()); + mic_manager_release(MicClientVoiceDictation); + + cl_assert(!s_mic_running); + cl_assert_equal_i(0, fake_event_get_count()); +} + +void test_mic_capture_service__start_read_stop(void) { + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); + cl_assert(mic_capture_service_is_active()); + cl_assert(s_mic_running); + cl_assert_equal_i(SPU, s_mic_buffer_len); + cl_assert_equal_i(MicClientAppCapture, mic_manager_get_owner()); + + // Second start is refused + cl_assert_equal_i(MicCaptureStartErrBusy, mic_capture_service_start(PebbleTask_App, SPU)); + + cl_assert_equal_i(0, mic_capture_service_get_available()); + + prv_deliver_chunk(7); + cl_assert_equal_i(SPU, mic_capture_service_get_available()); + cl_assert_equal_i(1, fake_event_get_count()); + PebbleEvent e = prv_last_event(); + cl_assert_equal_i(e.type, PEBBLE_MIC_CAPTURE_EVENT); + cl_assert_equal_i(e.mic_capture.type, MicCaptureEventData); + cl_assert_equal_i(e.mic_capture.num_samples, SPU); + cl_assert_equal_b(e.mic_capture.overrun, false); + + // More chunks before the app reads are coalesced into the pending event + prv_deliver_chunk(8); + prv_deliver_chunk(9); + cl_assert_equal_i(1, fake_event_get_count()); + cl_assert_equal_i(3 * SPU, mic_capture_service_get_available()); + + // Wrong task cannot read + int16_t out[SPU]; + cl_assert_equal_i(0, mic_capture_service_read(PebbleTask_Worker, out, SPU)); + + cl_assert_equal_i(SPU, mic_capture_service_read(PebbleTask_App, out, SPU)); + cl_assert_equal_i(7, out[0]); + cl_assert_equal_i(7, out[SPU - 1]); + cl_assert_equal_i(SPU, mic_capture_service_read(PebbleTask_App, out, SPU)); + cl_assert_equal_i(8, out[0]); + // Short read at the end + cl_assert_equal_i(SPU, mic_capture_service_read(PebbleTask_App, out, 2 * SPU)); + cl_assert_equal_i(9, out[0]); + cl_assert_equal_i(0, mic_capture_service_read(PebbleTask_App, out, SPU)); + + // After a read, the next chunk posts a new event + prv_deliver_chunk(1); + cl_assert_equal_i(2, fake_event_get_count()); + + // App-initiated stop: no event + mic_capture_service_stop(PebbleTask_Worker); // not the owner + cl_assert(mic_capture_service_is_active()); + mic_capture_service_stop(PebbleTask_App); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + cl_assert_equal_i(MicClientNone, mic_manager_get_owner()); + cl_assert_equal_i(2, fake_event_get_count()); + cl_assert_equal_i(0, mic_capture_service_read(PebbleTask_App, out, SPU)); +} + +void test_mic_capture_service__overrun_drops_newest(void) { + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); + const int chunks_that_fit = MIC_CAPTURE_RING_SAMPLES / SPU; + for (int i = 0; i < chunks_that_fit; i++) { + prv_deliver_chunk(i); + } + cl_assert_equal_i(MIC_CAPTURE_RING_SAMPLES, mic_capture_service_get_available()); + prv_deliver_chunk(99); // dropped + cl_assert_equal_i(MIC_CAPTURE_RING_SAMPLES, mic_capture_service_get_available()); + + // Drain, the dropped chunk is not there + int16_t out[SPU]; + for (int i = 0; i < chunks_that_fit; i++) { + cl_assert_equal_i(SPU, mic_capture_service_read(PebbleTask_App, out, SPU)); + cl_assert_equal_i(i, out[0]); + } + cl_assert_equal_i(0, mic_capture_service_get_available()); + + // The overrun is reported on the next data event + prv_deliver_chunk(1); + PebbleEvent e = prv_last_event(); + cl_assert_equal_i(e.mic_capture.type, MicCaptureEventData); + cl_assert_equal_b(e.mic_capture.overrun, true); + cl_assert_equal_i(SPU, mic_capture_service_read(PebbleTask_App, out, SPU)); + prv_deliver_chunk(2); + e = prv_last_event(); + cl_assert_equal_b(e.mic_capture.overrun, false); +} + +void test_mic_capture_service__focus_lost(void) { + // Idle: nothing happens + mic_capture_service_handle_app_focus_lost(); + cl_assert_equal_i(0, fake_event_get_count()); + + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); + mic_capture_service_handle_app_focus_lost(); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + cl_assert_equal_i(MicClientNone, mic_manager_get_owner()); + prv_assert_stopped_event(MicCaptureStopReasonFocusLost); +} + +void test_mic_capture_service__permission_revoked(void) { + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); + + // Still granted: nothing happens + mic_capture_service_handle_permission_changed(); + cl_assert(mic_capture_service_is_active()); + cl_assert_equal_i(0, fake_event_get_count()); + + s_permission_state = AppPermissionStateDenied; + mic_capture_service_handle_permission_changed(); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + prv_assert_stopped_event(MicCaptureStopReasonPermissionRevoked); +} + +void test_mic_capture_service__preempted_by_dictation(void) { + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); + prv_deliver_chunk(3); + + cl_assert(prv_start_dictation()); + cl_assert(!mic_capture_service_is_active()); + cl_assert_equal_i(MicClientVoiceDictation, mic_manager_get_owner()); + cl_assert(s_mic_running); + cl_assert(s_mic_buffer == s_dictation_buffer); + prv_assert_stopped_event(MicCaptureStopReasonPreempted); + + // Late reads after preemption yield nothing + int16_t out[SPU]; + cl_assert_equal_i(0, mic_capture_service_read(PebbleTask_App, out, SPU)); + cl_assert_equal_i(0, mic_capture_service_get_available()); +} + +void test_mic_capture_service__stop_for_task(void) { + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); + prv_deliver_chunk(3); + const uint32_t events = fake_event_get_count(); + + mic_capture_service_stop_for_task(PebbleTask_Worker); + cl_assert(mic_capture_service_is_active()); + + mic_capture_service_stop_for_task(PebbleTask_App); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + cl_assert_equal_i(events, fake_event_get_count()); + + // Can start again afterwards + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); +} From 79b1b0026f34f7726f577b247dd798fb111e4e95 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:48:56 +0200 Subject: [PATCH 08/15] popups: add OS mic banner A transparent, unfocusable strip at the bottom of the screen, pushed at discreet modal priority for as long as an app captures the microphone. Like Timeline Peek it reserves its area through the unobstructed area service, and the two obstructions compose so an app sees the union. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- src/fw/applib/unobstructed_area_service.h | 11 +- src/fw/popups/mic_banner.c | 224 ++++++++++++++++++ src/fw/popups/mic_banner.h | 31 +++ src/fw/popups/timeline/peek.c | 19 +- src/fw/process_management/app_manager.c | 5 +- .../mic_capture/mic_capture_service.c | 13 + src/fw/shell/normal/shell_event_loop.c | 4 + src/fw/shell/prf/stubs.c | 4 + tests/fw/services/test_mic_capture_service.c | 2 + tests/fw/test_app_manager.c | 1 + tests/fw/ui/test_timeline_peek.c | 1 + tests/stubs/stubs_mic_banner.h | 24 ++ 12 files changed, 327 insertions(+), 12 deletions(-) create mode 100644 src/fw/popups/mic_banner.c create mode 100644 src/fw/popups/mic_banner.h create mode 100644 tests/stubs/stubs_mic_banner.h diff --git a/src/fw/applib/unobstructed_area_service.h b/src/fw/applib/unobstructed_area_service.h index e247ccb104..086795938e 100644 --- a/src/fw/applib/unobstructed_area_service.h +++ b/src/fw/applib/unobstructed_area_service.h @@ -12,12 +12,11 @@ //! //! \brief Events about when the app's unobstructed area changes for visually adapting //! -//! Unobstructed Area enables a watchface to adapt to overlays partially obstructing it. Timeline -//! Peek is the only overlay, and it partially obstructs the bottom of watchfaces, displaying the -//! immediately upcoming or newly began event. Unobstructed Area is for use only with watchfaces. -//! There will be no Unobstructed Area events emitted for apps that are not watchfaces. Timeline -//! Peek is also limited to rectangular platforms, thus using Unobstructed Area on Chalk will also -//! result in no events. +//! Unobstructed Area enables an app to adapt to overlays partially obstructing it. Timeline Peek +//! partially obstructs the bottom of watchfaces, displaying the immediately upcoming or newly began +//! event; it is limited to rectangular platforms and watchfaces. The system "Listening" banner +//! obstructs the bottom of any app while it captures the microphone (see the Microphone API). +//! Apps that use neither receive no Unobstructed Area events. //! //! Watchfaces are encouraged to respect Unobstructed Area in order to dynamically resize within //! the remaining screen area that isn't obstructed by an overlay. Unobstructed Area provides diff --git a/src/fw/popups/mic_banner.c b/src/fw/popups/mic_banner.c new file mode 100644 index 0000000000..2c68aac372 --- /dev/null +++ b/src/fw/popups/mic_banner.c @@ -0,0 +1,224 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "popups/mic_banner.h" + +#include "applib/fonts/fonts.h" +#include "applib/graphics/graphics.h" +#include "applib/graphics/graphics_circle.h" +#include "applib/graphics/text.h" +#include "applib/ui/animation_interpolate.h" +#include "applib/ui/property_animation.h" +#include "applib/ui/window.h" +#include "applib/ui/window_stack.h" +#include "applib/unobstructed_area_service.h" +#include "kernel/ui/modals/modal_manager.h" +#include "pbl/services/i18n/i18n.h" +#include "pbl/util/math.h" +#include "popups/timeline/peek.h" +#include "process_management/app_manager.h" + +#define FRAME_VISIBLE GRect(0, DISP_ROWS - MIC_BANNER_HEIGHT, DISP_COLS, MIC_BANNER_HEIGHT) +#define FRAME_HIDDEN GRect(0, DISP_ROWS, DISP_COLS, MIC_BANNER_HEIGHT) +#define DOT_RADIUS (4) +#define DOT_MARGIN (8) + +typedef struct { + Window window; + Layer strip; //!< The visible bar; its frame is what slides in and out + Animation *animation; //!< Currently running slide, if any + bool visible; //!< Target state: shown (or showing) vs hidden (or hiding) +} MicBanner; + +static MicBanner s_banner; + +// The window is transparent; only the strip draws. +static void prv_window_update_proc(Layer *layer, GContext *ctx) { +} + +static void prv_update_proc(Layer *layer, GContext *ctx) { + const GRect bounds = layer->bounds; + graphics_context_set_fill_color(ctx, GColorBlack); + graphics_fill_rect(ctx, &bounds); + + GFont font = fonts_get_system_font(FONT_KEY_GOTHIC_14_BOLD); + const char *text = i18n_get("Listening", &s_banner); + const int16_t text_height = 18; + const int16_t max_text_width = bounds.size.w - (3 * DOT_MARGIN) - (2 * DOT_RADIUS); +#if PBL_ROUND + // The strip sits in a chord: centre the content horizontally, and vertically on the visible + // segment's centroid rather than the strip's middle, which is mostly off screen. + const GSize text_size = graphics_text_layout_get_max_used_size( + ctx, text, font, GRect(0, 0, max_text_width, text_height), GTextOverflowModeTrailingEllipsis, + GTextAlignmentLeft, NULL); + const int16_t group_width = (2 * DOT_RADIUS) + DOT_MARGIN + text_size.w; + const int16_t x0 = (bounds.size.w - group_width) / 2; + const int16_t centre_y = (bounds.size.h * 2) / 5; +#else + const int16_t x0 = DOT_MARGIN; + const int16_t centre_y = bounds.size.h / 2; +#endif + + const GPoint dot = GPoint(x0 + DOT_RADIUS, centre_y); + graphics_context_set_fill_color(ctx, PBL_IF_COLOR_ELSE(GColorRed, GColorWhite)); + graphics_fill_circle(ctx, dot, DOT_RADIUS); + + // Gothic 14 renders its caps in the top ~9 rows of the box; this puts them on the dot's centre. + const GRect text_box = GRect(dot.x + DOT_RADIUS + DOT_MARGIN, centre_y - (text_height / 2), + max_text_width, text_height); + graphics_context_set_text_color(ctx, GColorWhite); + graphics_draw_text(ctx, text, font, text_box, GTextOverflowModeTrailingEllipsis, + GTextAlignmentLeft, NULL); +} + +// Obstruction +//////////////////////////////////////////////////////////////////////////////// + +//! Framebuffer-space y where a strip whose top is at display row `strip_y` starts obstructing the +//! app, or the framebuffer height if it doesn't overlap. +static int16_t prv_obstruction_for_strip_y(int16_t strip_y) { + GSize fb_size; + app_manager_get_framebuffer_size(&fb_size); + if (fb_size.h != DISP_ROWS) { + // Legacy-sized apps predate the microphone API; don't report an obstruction to them. + return fb_size.h; + } + return MIN(strip_y, DISP_ROWS); +} + +//! The app sees the union of the banner and Timeline Peek. +static int16_t prv_compose(int16_t banner_y) { + return MIN(banner_y, timeline_peek_get_obstruction_origin_y()); +} + +int16_t mic_banner_get_obstruction_origin_y(void) { + return prv_obstruction_for_strip_y(s_banner.strip.frame.origin.y); +} + +// Slide animation (mirrors Timeline Peek) +//////////////////////////////////////////////////////////////////////////////// + +static void prv_frame_setup(Animation *animation) { + PropertyAnimation *prop_anim = (PropertyAnimation *)animation; + GRect from, to; + property_animation_get_from_grect(prop_anim, &from); + property_animation_get_to_grect(prop_anim, &to); + unobstructed_area_service_will_change(prv_compose(prv_obstruction_for_strip_y(from.origin.y)), + prv_compose(prv_obstruction_for_strip_y(to.origin.y))); +} + +static void prv_frame_update(Animation *animation, AnimationProgress progress) { + PropertyAnimation *prop_anim = (PropertyAnimation *)animation; + property_animation_update_grect(prop_anim, progress); + GRect to; + property_animation_get_to_grect(prop_anim, &to); + unobstructed_area_service_change(prv_compose(mic_banner_get_obstruction_origin_y()), + prv_compose(prv_obstruction_for_strip_y(to.origin.y)), progress); +} + +static void prv_frame_teardown(Animation *animation) { + PropertyAnimation *prop_anim = (PropertyAnimation *)animation; + GRect to; + property_animation_get_to_grect(prop_anim, &to); + unobstructed_area_service_did_change(prv_compose(prv_obstruction_for_strip_y(to.origin.y))); +} + +static GRect prv_frame_getter(void *subject) { + MicBanner *banner = subject; + GRect frame; + layer_get_frame(&banner->strip, &frame); + return frame; +} + +static void prv_frame_setter(void *subject, GRect frame) { + MicBanner *banner = subject; + layer_set_frame(&banner->strip, &frame); +} + +static const PropertyAnimationImplementation s_slide_impl = { + .base = + { + .setup = prv_frame_setup, + .update = prv_frame_update, + .teardown = prv_frame_teardown, + }, + .accessors = { + .getter.grect = prv_frame_getter, + .setter.grect = prv_frame_setter, + }, +}; + +static void prv_slide_stopped(Animation *animation, bool finished, void *context) { + MicBanner *banner = &s_banner; + banner->animation = NULL; + if (finished && !banner->visible) { + window_stack_remove(&banner->window, false /* animated */); + i18n_free_all(banner); + } +} + +static const AnimationHandlers s_slide_handlers = { + .stopped = prv_slide_stopped, +}; + +static void prv_slide_to(MicBanner *banner, GRect *to_frame) { + if (banner->animation) { + animation_unschedule(banner->animation); + banner->animation = NULL; + } + PropertyAnimation *prop_anim = property_animation_create(&s_slide_impl, banner, NULL, NULL); + property_animation_set_from_grect(prop_anim, &banner->strip.frame); + property_animation_set_to_grect(prop_anim, to_frame); + Animation *animation = property_animation_get_animation(prop_anim); + animation_set_duration(animation, interpolate_moook_duration()); + animation_set_custom_interpolation(animation, interpolate_moook); + animation_set_handlers(animation, s_slide_handlers, NULL); + banner->animation = animation; + animation_schedule(animation); +} + +// Public API +//////////////////////////////////////////////////////////////////////////////// + +void mic_banner_init(void) { + MicBanner *banner = &s_banner; + *banner = (MicBanner){}; + window_init(&banner->window, WINDOW_NAME("Mic Banner")); + window_set_focusable(&banner->window, false); + window_set_transparent(&banner->window, true); + layer_set_update_proc(&banner->window.layer, prv_window_update_proc); + GRect hidden = FRAME_HIDDEN; + layer_init(&banner->strip, &hidden); + layer_set_update_proc(&banner->strip, prv_update_proc); + layer_add_child(&banner->window.layer, &banner->strip); +} + +void mic_banner_show(void) { + MicBanner *banner = &s_banner; + if (banner->visible) { + return; + } + banner->visible = true; + if (!banner->animation) { + // Fresh show: start off screen, then slide up. A show during a hide just reverses. + GRect hidden = FRAME_HIDDEN; + layer_set_frame(&banner->strip, &hidden); + modal_window_push(&banner->window, ModalPriorityDiscreet, false /* animated */); + } + GRect visible = FRAME_VISIBLE; + prv_slide_to(banner, &visible); +} + +void mic_banner_hide(void) { + MicBanner *banner = &s_banner; + if (!banner->visible) { + return; + } + banner->visible = false; + GRect hidden = FRAME_HIDDEN; + prv_slide_to(banner, &hidden); +} + +bool mic_banner_is_visible(void) { + return s_banner.visible; +} diff --git a/src/fw/popups/mic_banner.h b/src/fw/popups/mic_banner.h new file mode 100644 index 0000000000..777b4de25d --- /dev/null +++ b/src/fw/popups/mic_banner.h @@ -0,0 +1,31 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include +#include + +//! OS-owned "listening" banner shown while an app captures the microphone. +//! +//! A transparent, unfocusable modal strip at the bottom of the screen. It reserves part of the +//! app's screen through the unobstructed area service, the same way Timeline Peek does, so an app +//! can lay out around it. The app cannot cover or dismiss it: capture stops as soon as the app +//! loses focus to any other modal window. + +// Round displays need a taller strip so the chord is wide enough for the centred content +#define MIC_BANNER_HEIGHT (PBL_IF_RECT_ELSE(24, 40)) + +void mic_banner_init(void); + +//! Shows the banner. Must be called on KernelMain. +void mic_banner_show(void); + +//! Hides the banner. Must be called on KernelMain. +void mic_banner_hide(void); + +bool mic_banner_is_visible(void); + +//! Y coordinate, in app framebuffer space, where the banner starts obstructing the app, or the +//! framebuffer height when the banner is hidden or does not overlap the app. +int16_t mic_banner_get_obstruction_origin_y(void); diff --git a/src/fw/popups/timeline/peek.c b/src/fw/popups/timeline/peek.c index 37c28d4d6d..6580dd780f 100644 --- a/src/fw/popups/timeline/peek.c +++ b/src/fw/popups/timeline/peek.c @@ -7,6 +7,7 @@ #include "process_management/app_manager.h" #include "applib/ui/window_stack.h" #include "applib/unobstructed_area_service.h" +#include "popups/mic_banner.h" #include "apps/system/timeline/common.h" #include "kernel/event_loop.h" #include "kernel/pbl_malloc.h" @@ -195,6 +196,11 @@ static int16_t prv_scale_y_to_framebuffer(int16_t display_y) { return (display_y * app_framebuffer_size.h) / DISP_ROWS; } +//! The mic banner may also obstruct the app; the app sees the union of both. +static int16_t prv_compose_obstruction_y(int16_t peek_y) { + return MIN(peek_y, mic_banner_get_obstruction_origin_y()); +} + static void prv_peek_frame_setup(Animation *animation) { PropertyAnimation *prop_anim = (PropertyAnimation *)animation; TimelinePeek *peek; @@ -204,8 +210,9 @@ static void prv_peek_frame_setup(Animation *animation) { GRect to_frame; property_animation_get_to_grect(prop_anim, &to_frame); if (prv_should_use_unobstructed_area()) { - unobstructed_area_service_will_change(prv_scale_y_to_framebuffer(from_frame.origin.y), - prv_scale_y_to_framebuffer(to_frame.origin.y)); + unobstructed_area_service_will_change( + prv_compose_obstruction_y(prv_scale_y_to_framebuffer(from_frame.origin.y)), + prv_compose_obstruction_y(prv_scale_y_to_framebuffer(to_frame.origin.y))); } } @@ -217,8 +224,9 @@ static void prv_peek_frame_update(Animation *animation, AnimationProgress progre GRect to_frame; property_animation_get_to_grect(prop_anim, &to_frame); if (prv_should_use_unobstructed_area()) { - unobstructed_area_service_change(prv_scale_y_to_framebuffer(peek->layout_layer.frame.origin.y), - prv_scale_y_to_framebuffer(to_frame.origin.y), progress); + unobstructed_area_service_change( + prv_compose_obstruction_y(prv_scale_y_to_framebuffer(peek->layout_layer.frame.origin.y)), + prv_compose_obstruction_y(prv_scale_y_to_framebuffer(to_frame.origin.y)), progress); } } @@ -227,7 +235,8 @@ static void prv_peek_frame_teardown(Animation *animation) { GRect to_frame; property_animation_get_to_grect(prop_anim, &to_frame); if (prv_should_use_unobstructed_area()) { - unobstructed_area_service_did_change(prv_scale_y_to_framebuffer(to_frame.origin.y)); + unobstructed_area_service_did_change( + prv_compose_obstruction_y(prv_scale_y_to_framebuffer(to_frame.origin.y))); } } diff --git a/src/fw/process_management/app_manager.c b/src/fw/process_management/app_manager.c index 53160b2cb7..2427d617c6 100644 --- a/src/fw/process_management/app_manager.c +++ b/src/fw/process_management/app_manager.c @@ -20,6 +20,7 @@ #include "pbl/mcu/privilege.h" #include "popups/health_tracking_ui.h" #include "popups/timeline/peek.h" +#include "popups/mic_banner.h" #include "process_management/app_run_state.h" #include "process_management/pebble_process_md.h" #include "process_management/process_heap.h" @@ -297,7 +298,9 @@ static bool prv_app_start(const PebbleProcessMd *app_md, const void *args, const ProcessAppSDKType sdk_type = process_metadata_get_app_sdk_type(app_md); // The rest of app_ram is available for app_state to use as it sees fit. - if (!app_state_configure(&app_ram, sdk_type, timeline_peek_get_obstruction_origin_y())) { + const int16_t obstruction_y = + MIN(timeline_peek_get_obstruction_origin_y(), mic_banner_get_obstruction_origin_y()); + if (!app_state_configure(&app_ram, sdk_type, obstruction_y)) { PBL_LOG_ERR("App state configuration failed"); return false; } diff --git a/src/fw/services/mic_capture/mic_capture_service.c b/src/fw/services/mic_capture/mic_capture_service.c index ebefe0d519..1d796fad4e 100644 --- a/src/fw/services/mic_capture/mic_capture_service.c +++ b/src/fw/services/mic_capture/mic_capture_service.c @@ -3,6 +3,7 @@ #include "pbl/services/mic_capture/mic_capture_service.h" +#include "kernel/event_loop.h" #include "kernel/events.h" #include "kernel/pbl_malloc.h" #include "kernel/ui/modals/modal_manager.h" @@ -10,6 +11,7 @@ #include "pbl/services/app_permissions/app_permissions.h" #include "pbl/services/mic_manager.h" #include "pbl/util/circular_buffer.h" +#include "popups/mic_banner.h" #include #include @@ -32,6 +34,15 @@ typedef struct { static PBL_MUTEX_DEFINE(s_lock); static MicCaptureState s_state; +// The banner is UI, so it is driven from KernelMain whichever task stops or starts capture. +static void prv_show_banner_cb(void *unused) { + mic_banner_show(); +} + +static void prv_hide_banner_cb(void *unused) { + mic_banner_hide(); +} + void mic_capture_service_init(void) { s_state = (MicCaptureState){}; } @@ -59,6 +70,7 @@ static void prv_teardown_locked(void) { kernel_free(s_state.chunk); s_state.ring_storage = NULL; s_state.chunk = NULL; + launcher_task_add_callback(prv_hide_banner_cb, NULL); } //! Stops capture for a system-originated reason and tells the app. `release_mic` is false when @@ -169,6 +181,7 @@ MicCaptureStartResult mic_capture_service_start(PebbleTask owner, uint16_t sampl return MicCaptureStartErrBusy; } + launcher_task_add_callback(prv_show_banner_cb, NULL); PBL_LOG_DBG("Capture started, %u samples per update", samples_per_update); return MicCaptureStartOk; } diff --git a/src/fw/shell/normal/shell_event_loop.c b/src/fw/shell/normal/shell_event_loop.c index d8182ab27b..c1986af2cc 100644 --- a/src/fw/shell/normal/shell_event_loop.c +++ b/src/fw/shell/normal/shell_event_loop.c @@ -11,6 +11,7 @@ #include "popups/bluetooth_pairing_ui.h" #include "popups/notifications/notification_window.h" #include "popups/timeline/peek.h" +#include "popups/mic_banner.h" #include "process_management/app_install_manager.h" #include "process_management/app_manager.h" #include "pbl/services/blob_db/api.h" @@ -58,6 +59,9 @@ void shell_event_loop_init(void) { app_message_sender_init(); watchface_init(); timeline_peek_init(); +#ifdef CONFIG_SERVICE_MIC_CAPTURE + mic_banner_init(); +#endif // Start activity tracking if enabled if (activity_prefs_tracking_is_enabled()) { activity_start_tracking(false /*test_mode*/); diff --git a/src/fw/shell/prf/stubs.c b/src/fw/shell/prf/stubs.c index dd66c54fe8..fafa9b0447 100644 --- a/src/fw/shell/prf/stubs.c +++ b/src/fw/shell/prf/stubs.c @@ -295,6 +295,10 @@ int16_t timeline_peek_get_obstruction_origin_y(void) { return DISP_ROWS; } +int16_t mic_banner_get_obstruction_origin_y(void) { + return DISP_ROWS; +} + void timeline_peek_handle_process_start(void) { } diff --git a/tests/fw/services/test_mic_capture_service.c b/tests/fw/services/test_mic_capture_service.c index babc7dbef0..96b306d8f9 100644 --- a/tests/fw/services/test_mic_capture_service.c +++ b/tests/fw/services/test_mic_capture_service.c @@ -21,6 +21,8 @@ //////////////////////////////////////////////////////////////// #include "stubs_logging.h" #include "stubs_passert.h" +#include "stubs_event_loop.h" +#include "stubs_mic_banner.h" // Fake mic driver, driven by the tests //////////////////////////////////////////////////////////////// diff --git a/tests/fw/test_app_manager.c b/tests/fw/test_app_manager.c index 71a5d8450b..f0e42bcbc9 100644 --- a/tests/fw/test_app_manager.c +++ b/tests/fw/test_app_manager.c @@ -55,6 +55,7 @@ #include "stubs_thread.h" #include "stubs_tick.h" #include "stubs_timeline_peek.h" +#include "stubs_mic_banner.h" #include "stubs_worker_manager.h" // Fake "Apps" diff --git a/tests/fw/ui/test_timeline_peek.c b/tests/fw/ui/test_timeline_peek.c index f43ab4f5f3..b1b16b8e0a 100644 --- a/tests/fw/ui/test_timeline_peek.c +++ b/tests/fw/ui/test_timeline_peek.c @@ -74,6 +74,7 @@ void clock_get_until_time(char *buffer, int buf_size, time_t timestamp, int max_ #include "stubs_unobstructed_area.h" #include "stubs_window_manager.h" #include "stubs_window_stack.h" +#include "stubs_mic_banner.h" // Helper Functions ///////////////////// diff --git a/tests/stubs/stubs_mic_banner.h b/tests/stubs/stubs_mic_banner.h new file mode 100644 index 0000000000..494fb2ef65 --- /dev/null +++ b/tests/stubs/stubs_mic_banner.h @@ -0,0 +1,24 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "popups/mic_banner.h" +#include "pbl/util/attributes.h" + +void WEAK mic_banner_init(void) { +} + +void WEAK mic_banner_show(void) { +} + +void WEAK mic_banner_hide(void) { +} + +bool WEAK mic_banner_is_visible(void) { + return false; +} + +int16_t WEAK mic_banner_get_obstruction_origin_y(void) { + return DISP_ROWS; +} From c247292c7960d594755b7921bab747de36b5d539 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:51:11 +0200 Subject: [PATCH 09/15] applib: add Microphone API mic_data_service_subscribe() delivers batches of raw 16 kHz mono PCM to a foreground app that declares and was granted the microphone permission, draining the kernel capture ring through syscalls. Capture ends with a MicDataStopReason when the app loses focus, the grant is revoked, or the system takes the microphone. Exported as SDK revision 111. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- src/fw/applib/mic_data_service.c | 131 ++++++++++++++++++ src/fw/applib/mic_data_service.h | 101 ++++++++++++++ src/fw/applib/mic_data_service_private.h | 18 +++ .../process_management/pebble_process_info.h | 3 +- src/fw/process_state/app_state/app_state.c | 8 ++ src/fw/process_state/app_state/app_state.h | 3 + src/fw/syscall/syscall.h | 7 + src/fw/syscall/syscall_mic_capture.c | 54 ++++++++ .../generate_native_sdk/exported_symbols.json | 49 ++++++- 9 files changed, 371 insertions(+), 3 deletions(-) create mode 100644 src/fw/applib/mic_data_service.c create mode 100644 src/fw/applib/mic_data_service.h create mode 100644 src/fw/applib/mic_data_service_private.h create mode 100644 src/fw/syscall/syscall_mic_capture.c diff --git a/src/fw/applib/mic_data_service.c b/src/fw/applib/mic_data_service.c new file mode 100644 index 0000000000..29fc9b8bef --- /dev/null +++ b/src/fw/applib/mic_data_service.c @@ -0,0 +1,131 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "applib/mic_data_service.h" +#include "applib/mic_data_service_private.h" + +#include "applib/applib_malloc.auto.h" +#include "kernel/events.h" +#include "kernel/pebble_tasks.h" +#include "pbl/services/mic_capture/mic_capture_service.h" +#include "process_state/app_state/app_state.h" +#include "syscall/syscall.h" + +static MicDataServiceState *prv_get_state(void) { + if (pebble_task_get_current() != PebbleTask_App) { + return NULL; + } + return app_state_get_mic_data_service_state(); +} + +static void prv_teardown(MicDataServiceState *state) { + event_service_client_unsubscribe(&state->event_info); + applib_free(state->buffer); + state->buffer = NULL; + state->handlers = (MicDataHandlers){}; + state->context = NULL; + state->active = false; +} + +static MicDataStopReason prv_map_stop_reason(uint8_t kernel_reason) { + switch ((MicCaptureStopReason)kernel_reason) { + case MicCaptureStopReasonStopped: + return MicDataStopReasonStopped; + case MicCaptureStopReasonFocusLost: + return MicDataStopReasonFocusLost; + case MicCaptureStopReasonPreempted: + return MicDataStopReasonInterrupted; + case MicCaptureStopReasonPermissionRevoked: + return MicDataStopReasonPermissionRevoked; + case MicCaptureStopReasonAppExit: + case MicCaptureStopReasonError: + break; + } + return MicDataStopReasonError; +} + +static void prv_handle_event(PebbleEvent *e, void *context) { + MicDataServiceState *state = context; + if (!state->active) { + return; + } + + if (e->mic_capture.type == MicCaptureEventStopped) { + const MicDataStopReason reason = prv_map_stop_reason(e->mic_capture.stop_reason); + const MicDataStoppedHandler stopped = state->handlers.stopped; + void *ctx = state->context; + prv_teardown(state); + if (stopped) { + stopped(reason, ctx); + } + return; + } + + bool overrun = e->mic_capture.overrun; + while (state->active && (sys_mic_capture_get_available() >= state->samples_per_update)) { + const uint32_t num_samples = sys_mic_capture_read(state->buffer, state->samples_per_update); + if (num_samples == 0) { + break; + } + state->handlers.data(state->buffer, num_samples, overrun, state->context); + overrun = false; + } +} + +MicDataStartResult mic_data_service_subscribe(uint32_t samples_per_update, MicDataHandlers handlers, + void *context) { + MicDataServiceState *state = prv_get_state(); + if (!state) { + return MicDataStartErrNotForeground; + } + if (!handlers.data || (samples_per_update < MIC_DATA_MIN_SAMPLES_PER_UPDATE) || + (samples_per_update > MIC_DATA_MAX_SAMPLES_PER_UPDATE)) { + return MicDataStartErrInvalidArgs; + } + if (state->active) { + return MicDataStartErrBusy; + } + + int16_t *buffer = applib_malloc(samples_per_update * sizeof(int16_t)); + if (!buffer) { + return MicDataStartErrNoMemory; + } + + const MicDataStartResult rv = (MicDataStartResult)sys_mic_capture_start(samples_per_update); + if (rv != MicDataStartOk) { + applib_free(buffer); + return rv; + } + + state->buffer = buffer; + state->samples_per_update = samples_per_update; + state->handlers = handlers; + state->context = context; + state->active = true; + event_service_client_subscribe(&state->event_info); + return MicDataStartOk; +} + +void mic_data_service_unsubscribe(void) { + MicDataServiceState *state = prv_get_state(); + if (!state || !state->active) { + return; + } + sys_mic_capture_stop(); + prv_teardown(state); +} + +bool mic_data_service_is_active(void) { + MicDataServiceState *state = prv_get_state(); + return state ? state->active : false; +} + +void mic_data_service_state_init(MicDataServiceState *state) { + *state = (MicDataServiceState){ + .event_info = { + .type = PEBBLE_MIC_CAPTURE_EVENT, + .handler = prv_handle_event, + .context = state, + }, + }; +} diff --git a/src/fw/applib/mic_data_service.h b/src/fw/applib/mic_data_service.h new file mode 100644 index 0000000000..48bf2aae4b --- /dev/null +++ b/src/fw/applib/mic_data_service.h @@ -0,0 +1,101 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include +#include + +//! @addtogroup Foundation +//! @{ +//! @addtogroup Microphone +//! \brief Live PCM audio from the watch microphone +//! +//! The Microphone API delivers raw 16 kHz, 16-bit mono PCM to your app while it is in the +//! foreground. It is for watchapps only: a watchface is never allowed to record, whatever its +//! manifest declares. To use it, declare `capabilities: ["microphone"]` in your app's manifest; the +//! user is asked to grant the permission in the phone app and can revoke it at any time (see the +//! Permissions API). While your app captures, the system shows a "Listening" banner at the bottom +//! of the screen which reduces your unobstructed area (see \ref layer_get_unobstructed_bounds). +//! +//! Capture ends on its own, with a \ref MicDataStopReason, whenever your app loses focus (a +//! notification, alert, or the dictation UI), the permission is revoked, or the system needs the +//! microphone. Your app owns everything after capture: processing, encoding, buffering and +//! delivery to the phone. +//! @{ + +//! Sample rate of the delivered audio, in Hz. +#define MIC_DATA_SAMPLE_RATE (16000) + +//! Result of \ref mic_data_service_subscribe. +typedef enum MicDataStartResult { + //! Capture started + MicDataStartOk = 0, + //! The app does not declare the microphone capability in its manifest + MicDataStartErrNotDeclared, + //! The user has not granted the microphone permission + MicDataStartErrPermissionDenied, + //! The microphone is in use (by this app or the system) + MicDataStartErrBusy, + //! The app is not in the foreground + MicDataStartErrNotForeground, + //! Invalid arguments (see the limits on samples_per_update) + MicDataStartErrInvalidArgs, + //! Not enough memory to start capture + MicDataStartErrNoMemory, + //! Watchfaces can never record + MicDataStartErrWatchface, +} MicDataStartResult; + +//! Why capture ended. +typedef enum MicDataStopReason { + //! The app unsubscribed + MicDataStopReasonStopped = 0, + //! The app lost focus, e.g. a notification appeared + MicDataStopReasonFocusLost, + //! The system took the microphone, e.g. for dictation + MicDataStopReasonInterrupted, + //! The user revoked the microphone permission + MicDataStopReasonPermissionRevoked, + //! An unexpected error + MicDataStopReasonError, +} MicDataStopReason; + +//! Handler receiving a batch of samples. +//! @param samples samples_per_update signed 16-bit mono samples. Only valid during the call. +//! @param num_samples Number of samples in the batch +//! @param overrun true if samples were dropped before this batch because the app fell behind +//! @param context The context passed to \ref mic_data_service_subscribe +typedef void (*MicDataHandler)(const int16_t *samples, uint32_t num_samples, bool overrun, + void *context); + +//! Handler called when capture ends for a reason other than the app unsubscribing. The +//! subscription is gone by the time it is called. +typedef void (*MicDataStoppedHandler)(MicDataStopReason reason, void *context); + +typedef struct MicDataHandlers { + MicDataHandler data; + MicDataStoppedHandler stopped; +} MicDataHandlers; + +//! Minimum number of samples per update (5 ms). +#define MIC_DATA_MIN_SAMPLES_PER_UPDATE (80) +//! Maximum number of samples per update (100 ms). +#define MIC_DATA_MAX_SAMPLES_PER_UPDATE (1600) + +//! Starts capturing and subscribes to batches of samples. +//! @param samples_per_update Batch size, between \ref MIC_DATA_MIN_SAMPLES_PER_UPDATE and +//! \ref MIC_DATA_MAX_SAMPLES_PER_UPDATE. Choose your encoder's frame size. +//! @param handlers The handlers to call; `data` is required +//! @param context User-provided context passed to the handlers +MicDataStartResult mic_data_service_subscribe(uint32_t samples_per_update, MicDataHandlers handlers, + void *context); + +//! Stops capturing. Safe to call when not capturing. +void mic_data_service_unsubscribe(void); + +//! @return true while the app is capturing +bool mic_data_service_is_active(void); + +//! @} // end addtogroup Microphone +//! @} // end addtogroup Foundation diff --git a/src/fw/applib/mic_data_service_private.h b/src/fw/applib/mic_data_service_private.h new file mode 100644 index 0000000000..28375e361a --- /dev/null +++ b/src/fw/applib/mic_data_service_private.h @@ -0,0 +1,18 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "applib/event_service_client.h" +#include "applib/mic_data_service.h" + +typedef struct MicDataServiceState { + EventServiceInfo event_info; + MicDataHandlers handlers; + void *context; + int16_t *buffer; //!< samples_per_update samples, owned by the subscription + uint32_t samples_per_update; + bool active; +} MicDataServiceState; + +void mic_data_service_state_init(MicDataServiceState *state); diff --git a/src/fw/process_management/pebble_process_info.h b/src/fw/process_management/pebble_process_info.h index 5503c9ca71..33b43d14db 100644 --- a/src/fw/process_management/pebble_process_info.h +++ b/src/fw/process_management/pebble_process_info.h @@ -171,9 +171,10 @@ typedef enum { // app_touch_navigation_enable() opt-in for third-party touch nav (rev 108) sdk.major:0x5 // .minor:0x6a -- Add HRV sampling API (health_service_set_hrv_sample_period) (rev 109) // sdk.major:0x5 .minor:0x6b -- Add app permissions API (rev 110) +// sdk.major:0x5 .minor:0x6c -- Add Microphone API (mic_data_service_subscribe) (rev 111) #define PROCESS_INFO_CURRENT_SDK_VERSION_MAJOR 0x5 -#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6b +#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6c // The first SDK to ship with 2.x APIs #define PROCESS_INFO_FIRST_2X_SDK_VERSION_MAJOR 0x4 diff --git a/src/fw/process_state/app_state/app_state.c b/src/fw/process_state/app_state/app_state.c index 58213d1d71..615941118d 100644 --- a/src/fw/process_state/app_state/app_state.c +++ b/src/fw/process_state/app_state/app_state.c @@ -83,6 +83,8 @@ typedef struct { AppPermissionServiceState app_permission_service_state; + MicDataServiceState mic_data_service_state; + HealthServiceState health_service_state; LocaleInfo locale_info; @@ -311,6 +313,8 @@ NOINLINE void app_state_init(void) { app_permission_service_state_init(app_state_get_app_permission_service_state()); + mic_data_service_state_init(app_state_get_mic_data_service_state()); + tick_timer_service_state_init(app_state_get_tick_timer_service_state()); touch_service_state_init(app_state_get_touch_service_state()); @@ -448,6 +452,10 @@ AppPermissionServiceState *app_state_get_app_permission_service_state(void) { return &s_app_state_ptr->app_permission_service_state; } +MicDataServiceState *app_state_get_mic_data_service_state(void) { + return &s_app_state_ptr->mic_data_service_state; +} + HealthServiceState *app_state_get_health_service_state(void) { return &s_app_state_ptr->health_service_state; } diff --git a/src/fw/process_state/app_state/app_state.h b/src/fw/process_state/app_state/app_state.h index 72281599a5..64c5308570 100644 --- a/src/fw/process_state/app_state/app_state.h +++ b/src/fw/process_state/app_state/app_state.h @@ -8,6 +8,7 @@ #include "applib/app_inbox.h" #include "applib/app_message/app_message_internal.h" #include "applib/app_permissions_private.h" +#include "applib/mic_data_service_private.h" #include "applib/app_wakeup.h" #include "applib/backlight_service_private.h" #include "applib/battery_state_service_private.h" @@ -119,6 +120,8 @@ ConnectionServiceState *app_state_get_connection_service_state(void); AppPermissionServiceState *app_state_get_app_permission_service_state(void); +MicDataServiceState *app_state_get_mic_data_service_state(void); + LocaleInfo *app_state_get_locale_info(void); ContentIndicatorsBuffer *app_state_get_content_indicators_buffer(void); diff --git a/src/fw/syscall/syscall.h b/src/fw/syscall/syscall.h index 8a72ac7dd2..84582cc72e 100644 --- a/src/fw/syscall/syscall.h +++ b/src/fw/syscall/syscall.h @@ -106,6 +106,13 @@ bool sys_speaker_is_muted(void); //! @return AppPermissionState for the running app uint8_t sys_app_permission_get_state(uint8_t permission); +//! Live microphone capture for the app task. See mic_capture_service.h. +uint8_t sys_mic_capture_start(uint16_t samples_per_update); +void sys_mic_capture_stop(void); +uint32_t sys_mic_capture_read(int16_t *out, uint32_t max_samples); +uint32_t sys_mic_capture_get_available(void); +bool sys_mic_capture_is_active(void); + void sys_get_app_uuid(Uuid *uuid); bool sys_app_is_watchface(void); AppInstallId sys_app_manager_get_current_app_id(void); diff --git a/src/fw/syscall/syscall_mic_capture.c b/src/fw/syscall/syscall_mic_capture.c new file mode 100644 index 0000000000..2f3469fbba --- /dev/null +++ b/src/fw/syscall/syscall_mic_capture.c @@ -0,0 +1,54 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "syscall/syscall.h" +#include "syscall/syscall_internal.h" + +#include "kernel/pebble_tasks.h" +#include "pbl/services/mic_capture/mic_capture_service.h" + +// Builds without the capture service (e.g. PRF) refuse every request. + +DEFINE_SYSCALL(uint8_t, sys_mic_capture_start, uint16_t samples_per_update) { +#ifdef CONFIG_SERVICE_MIC_CAPTURE + return (uint8_t)mic_capture_service_start(pebble_task_get_current(), samples_per_update); +#else + return (uint8_t)MicCaptureStartErrDenied; +#endif +} + +DEFINE_SYSCALL(void, sys_mic_capture_stop, void) { +#ifdef CONFIG_SERVICE_MIC_CAPTURE + mic_capture_service_stop(pebble_task_get_current()); +#endif +} + +DEFINE_SYSCALL(uint32_t, sys_mic_capture_read, int16_t *out, uint32_t max_samples) { + if (PRIVILEGE_WAS_ELEVATED) { + if (max_samples > MIC_CAPTURE_MAX_SAMPLES_PER_UPDATE) { + syscall_failed(); + } + syscall_assert_userspace_buffer(out, max_samples * sizeof(int16_t)); + } +#ifdef CONFIG_SERVICE_MIC_CAPTURE + return mic_capture_service_read(pebble_task_get_current(), out, max_samples); +#else + return 0; +#endif +} + +DEFINE_SYSCALL(uint32_t, sys_mic_capture_get_available, void) { +#ifdef CONFIG_SERVICE_MIC_CAPTURE + return mic_capture_service_get_available(); +#else + return 0; +#endif +} + +DEFINE_SYSCALL(bool, sys_mic_capture_is_active, void) { +#ifdef CONFIG_SERVICE_MIC_CAPTURE + return mic_capture_service_is_active(); +#else + return false; +#endif +} diff --git a/tools/generate_native_sdk/exported_symbols.json b/tools/generate_native_sdk/exported_symbols.json index 55dad92834..7d24d27aba 100644 --- a/tools/generate_native_sdk/exported_symbols.json +++ b/tools/generate_native_sdk/exported_symbols.json @@ -4,7 +4,7 @@ "You should also make sure you are obeying our API design guidelines:", "https://pebbletechnology.atlassian.net/wiki/display/DEV/SDK+API+Design+Guidelines" ], - "revision" : "110", + "revision" : "111", "version" : "2.0", "files": [ "include/pbl/drivers/ambient_light.h", @@ -137,7 +137,8 @@ "fw/applib/rockyjs/rocky.h", "fw/applib/unobstructed_area_service.h", "include/pbl/services/app_permissions/app_permissions_types.h", - "fw/applib/app_permissions.h" + "fw/applib/app_permissions.h", + "fw/applib/mic_data_service.h" ], "exports": [ { @@ -4781,6 +4782,50 @@ "addedRevision": "110" } ] + }, { + "type": "group", + "name": "Microphone", + "appOnly": true, + "addedRevision": "111", + "exports": [ + { + "type": "define", + "name": "MIC_DATA_SAMPLE_RATE" + }, { + "type": "define", + "name": "MIC_DATA_MIN_SAMPLES_PER_UPDATE" + }, { + "type": "define", + "name": "MIC_DATA_MAX_SAMPLES_PER_UPDATE" + }, { + "type": "type", + "name": "MicDataStartResult" + }, { + "type": "type", + "name": "MicDataStopReason" + }, { + "type": "type", + "name": "MicDataHandler" + }, { + "type": "type", + "name": "MicDataStoppedHandler" + }, { + "type": "type", + "name": "MicDataHandlers" + }, { + "type": "function", + "name": "mic_data_service_subscribe", + "addedRevision": "111" + }, { + "type": "function", + "name": "mic_data_service_unsubscribe", + "addedRevision": "111" + }, { + "type": "function", + "name": "mic_data_service_is_active", + "addedRevision": "111" + } + ] }, { "type": "group", "name": "Light", From 7b4b8556c3d3b4fb67b62ce4994399e4ed8f83a9 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:53:42 +0200 Subject: [PATCH 10/15] apps/demo: add mic demo app SELECT toggles live capture through the Microphone API. The screen shows the permission state, the unobstructed height (which shrinks under the OS banner), the RMS of the last batch and the overrun count, and logs why capture stopped. Enable with CONFIG_DEMO_APP_MIC_DEMO=y. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- src/fw/apps/demo/CMakeLists.txt | 3 + src/fw/apps/demo/Kconfig | 1 + src/fw/apps/demo/mic_demo/CMakeLists.txt | 6 + src/fw/apps/demo/mic_demo/Kconfig | 6 + src/fw/apps/demo/mic_demo/mic_demo.c | 173 ++++++++++++++++++ src/fw/apps/demo/mic_demo/mic_demo.h | 8 + .../normal/system_app_registry_list.json | 8 + 7 files changed, 205 insertions(+) create mode 100644 src/fw/apps/demo/mic_demo/CMakeLists.txt create mode 100644 src/fw/apps/demo/mic_demo/Kconfig create mode 100644 src/fw/apps/demo/mic_demo/mic_demo.c create mode 100644 src/fw/apps/demo/mic_demo/mic_demo.h diff --git a/src/fw/apps/demo/CMakeLists.txt b/src/fw/apps/demo/CMakeLists.txt index 628699178d..45ef086902 100644 --- a/src/fw/apps/demo/CMakeLists.txt +++ b/src/fw/apps/demo/CMakeLists.txt @@ -5,6 +5,9 @@ add_subdirectory(shared) if(CONFIG_DEMO_APP_ACCEL_DEMO) add_subdirectory(accel_demo) endif() +if(CONFIG_DEMO_APP_MIC_DEMO) + add_subdirectory(mic_demo) +endif() if(CONFIG_DEMO_APP_ACTION_MENU) add_subdirectory(action_menu) endif() diff --git a/src/fw/apps/demo/Kconfig b/src/fw/apps/demo/Kconfig index 30b972b9f2..0c3a0a77e9 100644 --- a/src/fw/apps/demo/Kconfig +++ b/src/fw/apps/demo/Kconfig @@ -18,6 +18,7 @@ rsource "dialogs/Kconfig" rsource "double_tap_test/Kconfig" rsource "emoji_test/Kconfig" rsource "event_service/Kconfig" +rsource "mic_demo/Kconfig" rsource "exit/Kconfig" rsource "flash_diagnostic/Kconfig" rsource "flash_prof/Kconfig" diff --git a/src/fw/apps/demo/mic_demo/CMakeLists.txt b/src/fw/apps/demo/mic_demo/CMakeLists.txt new file mode 100644 index 0000000000..d26d5e26ed --- /dev/null +++ b/src/fw/apps/demo/mic_demo/CMakeLists.txt @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +pbl_library() +pbl_library_sources(mic_demo.c) +pbl_library_include_directories(.) diff --git a/src/fw/apps/demo/mic_demo/Kconfig b/src/fw/apps/demo/mic_demo/Kconfig new file mode 100644 index 0000000000..bfc4f44555 --- /dev/null +++ b/src/fw/apps/demo/mic_demo/Kconfig @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +config DEMO_APP_MIC_DEMO + bool "Mic Demo" + depends on SERVICE_MIC_CAPTURE diff --git a/src/fw/apps/demo/mic_demo/mic_demo.c b/src/fw/apps/demo/mic_demo/mic_demo.c new file mode 100644 index 0000000000..3f86e6aed8 --- /dev/null +++ b/src/fw/apps/demo/mic_demo/mic_demo.c @@ -0,0 +1,173 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "mic_demo.h" + +#include "applib/app.h" +#include "applib/app_permissions.h" +#include "applib/mic_data_service.h" +#include "applib/ui/ui.h" +#include "kernel/pbl_malloc.h" +#include "process_state/app_state/app_state.h" +#include + +#include +#include + +// Exercises the Microphone API: SELECT toggles capture, the screen shows the permission state, +// the RMS level of the last batch and the unobstructed height (which shrinks under the banner). + +#define SAMPLES_PER_UPDATE (320) // 20 ms + +typedef struct { + Window window; + TextLayer status_layer; + TextLayer level_layer; + char status_buffer[64]; + char level_buffer[48]; + uint32_t batches; + uint32_t overruns; +} MicDemoAppData; + +static void prv_update_status(MicDemoAppData *data, const char *text) { + const AppPermissionState state = app_permission_get_state(AppPermission_Microphone); + GRect unobstructed; + layer_get_unobstructed_bounds(&data->window.layer, &unobstructed); + snprintf(data->status_buffer, sizeof(data->status_buffer), "perm=%u h=%d\n%s", (unsigned)state, + unobstructed.size.h, text); + text_layer_set_text(&data->status_layer, data->status_buffer); +} + +static void prv_data_handler(const int16_t *samples, uint32_t num_samples, bool overrun, + void *context) { + MicDemoAppData *data = context; + uint64_t acc = 0; + for (uint32_t i = 0; i < num_samples; i++) { + acc += (int32_t)samples[i] * (int32_t)samples[i]; + } + // Integer square root of the mean square + uint32_t mean = (uint32_t)(acc / num_samples); + uint32_t rms = 0; + for (uint32_t bit = 1u << 15; bit; bit >>= 1) { + const uint32_t candidate = rms | bit; + if (candidate * candidate <= mean) { + rms = candidate; + } + } + data->batches++; + if (overrun) { + data->overruns++; + } + snprintf(data->level_buffer, sizeof(data->level_buffer), + "rms %" PRIu32 "\nbatch %" PRIu32 " drop %" PRIu32, rms, data->batches, data->overruns); + text_layer_set_text(&data->level_layer, data->level_buffer); + if ((data->batches % 50) == 0) { + PBL_LOG_DBG("mic demo: %" PRIu32 " batches, rms %" PRIu32 ", overruns %" PRIu32, data->batches, + rms, data->overruns); + } +} + +static void prv_stopped_handler(MicDataStopReason reason, void *context) { + MicDemoAppData *data = context; + static const char *const s_reasons[] = { + [MicDataStopReasonStopped] = "stopped", + [MicDataStopReasonFocusLost] = "focus lost", + [MicDataStopReasonInterrupted] = "interrupted", + [MicDataStopReasonPermissionRevoked] = "revoked", + [MicDataStopReasonError] = "error", + }; + PBL_LOG_DBG("mic demo: capture stopped (%s)", s_reasons[reason]); + prv_update_status(data, s_reasons[reason]); +} + +static void prv_permission_changed(AppPermission permission, AppPermissionState state, + void *context) { + PBL_LOG_DBG("mic demo: permission %u -> %u", permission, state); + prv_update_status(context, "permission changed"); +} + +static void prv_select_click_handler(ClickRecognizerRef recognizer, void *context) { + MicDemoAppData *data = context; + if (mic_data_service_is_active()) { + mic_data_service_unsubscribe(); + prv_update_status(data, "idle"); + return; + } + const MicDataStartResult rv = mic_data_service_subscribe( + SAMPLES_PER_UPDATE, + (MicDataHandlers){.data = prv_data_handler, .stopped = prv_stopped_handler}, data); + static const char *const s_results[] = { + [MicDataStartOk] = "listening", + [MicDataStartErrNotDeclared] = "not declared", + [MicDataStartErrPermissionDenied] = "denied", + [MicDataStartErrBusy] = "busy", + [MicDataStartErrNotForeground] = "not foreground", + [MicDataStartErrInvalidArgs] = "invalid args", + [MicDataStartErrNoMemory] = "no memory", + [MicDataStartErrWatchface] = "watchface", + }; + PBL_LOG_DBG("mic demo: subscribe -> %s", s_results[rv]); + prv_update_status(data, s_results[rv]); +} + +static void prv_click_config_provider(void *context) { + window_single_click_subscribe(BUTTON_ID_SELECT, prv_select_click_handler); +} + +static void prv_unobstructed_did_change(void *context) { + MicDemoAppData *data = context; + prv_update_status(data, mic_data_service_is_active() ? "listening" : "idle"); +} + +static void prv_window_load(Window *window) { + MicDemoAppData *data = window_get_user_data(window); + const GRect bounds = window->layer.bounds; + + text_layer_init(&data->status_layer, &GRect(0, 10, bounds.size.w, 60)); + text_layer_set_text_alignment(&data->status_layer, GTextAlignmentCenter); + layer_add_child(&window->layer, &data->status_layer.layer); + + text_layer_init(&data->level_layer, &GRect(0, 70, bounds.size.w, 60)); + text_layer_set_text_alignment(&data->level_layer, GTextAlignmentCenter); + layer_add_child(&window->layer, &data->level_layer.layer); + + prv_update_status(data, "press SELECT"); +} + +static void prv_handle_init(void) { + MicDemoAppData *data = app_zalloc_check(sizeof(*data)); + app_state_set_user_data(data); + + window_init(&data->window, "Mic Demo"); + window_set_user_data(&data->window, data); + window_set_click_config_provider_with_context(&data->window, prv_click_config_provider, data); + window_set_window_handlers(&data->window, &(WindowHandlers){.load = prv_window_load}); + + app_permission_service_subscribe(prv_permission_changed, data); + app_unobstructed_area_service_subscribe( + (UnobstructedAreaHandlers){.did_change = prv_unobstructed_did_change}, data); + + app_window_stack_push(&data->window, true /* animated */); +} + +static void prv_handle_deinit(void) { + MicDemoAppData *data = app_state_get_user_data(); + mic_data_service_unsubscribe(); + app_permission_service_unsubscribe(); + app_unobstructed_area_service_unsubscribe(); + app_free(data); +} + +static void s_main(void) { + prv_handle_init(); + app_event_loop(); + prv_handle_deinit(); +} + +const PebbleProcessMd *mic_demo_get_info(void) { + static const PebbleProcessMdSystem s_mic_demo_info = { + .common.main_func = s_main, + .name = "Mic Demo", + }; + return (const PebbleProcessMd *)&s_mic_demo_info; +} diff --git a/src/fw/apps/demo/mic_demo/mic_demo.h b/src/fw/apps/demo/mic_demo/mic_demo.h new file mode 100644 index 0000000000..e0393ea87e --- /dev/null +++ b/src/fw/apps/demo/mic_demo/mic_demo.h @@ -0,0 +1,8 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "process_management/pebble_process_md.h" + +const PebbleProcessMd *mic_demo_get_info(void); diff --git a/src/fw/shell/normal/system_app_registry_list.json b/src/fw/shell/normal/system_app_registry_list.json index 8b841e0d0e..364bf262a9 100644 --- a/src/fw/shell/normal/system_app_registry_list.json +++ b/src/fw/shell/normal/system_app_registry_list.json @@ -283,6 +283,14 @@ "CONFIG_DEMO_APP_ACCEL_DEMO" ] }, + { + "id": -193, + "enum": "MIC_DEMO", + "md_fn": "mic_demo_get_info", + "ifdefs": [ + "CONFIG_DEMO_APP_MIC_DEMO" + ] + }, { "id": -43, "enum": "PERSIST", From 71e372050068f90b7c069db7c24040939d43116b Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:56:57 +0200 Subject: [PATCH 11/15] services: extract codec-agnostic audio_encoder from voice_speex Dictation's Speex wrapper becomes a backend of a small encoder service with a single, task-owned instance and a codec descriptor (AudioEncoderInfo) that tells the consumer how to decode. Dictation opens it as the system owner and takes precedence over an app. This is the seam for the upcoming app-facing encoder API and for an Opus/SILK backend. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- .../services/audio_encoder/audio_encoder.h | 50 ++++ .../audio_encoder/audio_encoder_backend.h | 27 ++ .../audio_encoder/audio_encoder_types.h | 35 +++ src/fw/process_management/app_manager.c | 8 +- src/fw/services/CMakeLists.txt | 1 + src/fw/services/Kconfig | 1 + src/fw/services/audio_encoder/CMakeLists.txt | 12 + src/fw/services/audio_encoder/Kconfig | 16 ++ src/fw/services/audio_encoder/audio_encoder.c | 114 +++++++++ .../audio_encoder/audio_encoder_backends.c | 17 ++ .../audio_encoder/audio_encoder_speex.c | 153 ++++++++++++ src/fw/services/services_normal/service.c | 4 + src/fw/services/voice/Kconfig | 1 + src/fw/services/voice/voice_speex.c | 233 +++--------------- tests/fw/services/CMakeLists.txt | 6 + tests/fw/services/test_audio_encoder.c | 178 +++++++++++++ 16 files changed, 660 insertions(+), 196 deletions(-) create mode 100644 include/pbl/services/audio_encoder/audio_encoder.h create mode 100644 include/pbl/services/audio_encoder/audio_encoder_backend.h create mode 100644 include/pbl/services/audio_encoder/audio_encoder_types.h create mode 100644 src/fw/services/audio_encoder/CMakeLists.txt create mode 100644 src/fw/services/audio_encoder/Kconfig create mode 100644 src/fw/services/audio_encoder/audio_encoder.c create mode 100644 src/fw/services/audio_encoder/audio_encoder_backends.c create mode 100644 src/fw/services/audio_encoder/audio_encoder_speex.c create mode 100644 tests/fw/services/test_audio_encoder.c diff --git a/include/pbl/services/audio_encoder/audio_encoder.h b/include/pbl/services/audio_encoder/audio_encoder.h new file mode 100644 index 0000000000..6712187c46 --- /dev/null +++ b/include/pbl/services/audio_encoder/audio_encoder.h @@ -0,0 +1,50 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "kernel/pebble_tasks.h" +#include "pbl/services/audio_encoder/audio_encoder_types.h" + +#include +#include + +//! Codec-agnostic speech encoder shared by dictation and the app Microphone API. +//! +//! One encoder instance exists at a time, owned by a task. Dictation opens it as the system +//! owner and takes it away from an app if needed (the app's capture has already been preempted +//! by then). Backends are selected at build time; see audio_encoder_backend.h. + +_Static_assert(sizeof(AudioEncoderInfo) == 16, "AudioEncoderInfo is part of the SDK ABI"); + +//! Largest input frame any backend accepts, in samples (all channels). +#define AUDIO_ENCODER_MAX_FRAME_SAMPLES (1600) +//! Largest encoded packet any backend produces. +#define AUDIO_ENCODER_MAX_PACKET_BYTES (512) + +//! Owner used by kernel clients (dictation). +#define AUDIO_ENCODER_SYSTEM_OWNER (PebbleTask_KernelMain) + +void audio_encoder_service_init(void); + +bool audio_encoder_service_is_codec_available(AudioCodec codec); + +//! Opens the encoder for `owner`. Fails if another task holds it, unless the caller is the +//! system owner, which closes an app's encoder first. +//! @param codec Codec to encode with +//! @param owner Task that will own the encoder +//! @param[out] info_out Filled on success +bool audio_encoder_service_open(AudioCodec codec, PebbleTask owner, AudioEncoderInfo *info_out); + +//! Encodes exactly frame_samples * channels samples. +//! @return number of bytes written to out, or a negative value on error +int audio_encoder_service_encode(PebbleTask owner, const int16_t *pcm, uint32_t num_samples, + uint8_t *out, uint32_t out_len); + +//! Closes the encoder if `owner` holds it. +void audio_encoder_service_close(PebbleTask owner); + +//! Closes the encoder if `task` holds it. For process cleanup. +void audio_encoder_service_close_for_task(PebbleTask task); + +bool audio_encoder_service_is_open(void); diff --git a/include/pbl/services/audio_encoder/audio_encoder_backend.h b/include/pbl/services/audio_encoder/audio_encoder_backend.h new file mode 100644 index 0000000000..64990ba279 --- /dev/null +++ b/include/pbl/services/audio_encoder/audio_encoder_backend.h @@ -0,0 +1,27 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "pbl/services/audio_encoder/audio_encoder.h" + +#include + +//! One codec implementation. A backend is a singleton: open() allocates its state, close() +//! frees it, and encode() is only called in between. +typedef struct AudioEncoderBackend { + AudioCodec codec; + //! @param[out] info_out Filled on success + bool (*open)(AudioEncoderInfo *info_out); + //! @param pcm frame_samples * channels samples + //! @return bytes written or negative on error + int (*encode)(const int16_t *pcm, uint8_t *out, uint32_t out_len); + void (*close)(void); +} AudioEncoderBackend; + +//! Backends compiled into this firmware. Tests provide their own. +const AudioEncoderBackend *const *audio_encoder_get_backends(size_t *num_backends_out); + +#ifdef CONFIG_SPEEX +extern const AudioEncoderBackend g_audio_encoder_backend_speex; +#endif diff --git a/include/pbl/services/audio_encoder/audio_encoder_types.h b/include/pbl/services/audio_encoder/audio_encoder_types.h new file mode 100644 index 0000000000..f4ce17ed9b --- /dev/null +++ b/include/pbl/services/audio_encoder/audio_encoder_types.h @@ -0,0 +1,35 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include + +//! @addtogroup Foundation +//! @{ +//! @addtogroup AudioEncoder +//! @{ + +//! Speech codecs. Availability depends on the firmware build; check with +//! \ref audio_encoder_codec_available. +typedef enum AudioCodec { + AudioCodecInvalid = 0, + //! Speex wideband, 16 kHz, ~9.8 kbps. Frames of 320 samples, ~30 byte packets. + AudioCodecSpeexWB = 1, + AudioCodecCount, +} AudioCodec; + +//! Describes an open encoder. Everything an app needs to tell the phone how to decode. +typedef struct AudioEncoderInfo { + uint8_t codec; //!< AudioCodec + uint8_t channels; //!< Interleaved input channels (1 for the watch mic) + uint16_t frame_samples; //!< Input samples per channel per encode call + uint32_t sample_rate; //!< Hz + uint32_t bitrate; //!< bits per second (nominal) + uint16_t max_packet_bytes; //!< Upper bound on one encoded frame + uint8_t bitstream_version; //!< Codec-specific bitstream version + uint8_t reserved; +} AudioEncoderInfo; + +//! @} // end addtogroup AudioEncoder +//! @} // end addtogroup Foundation diff --git a/src/fw/process_management/app_manager.c b/src/fw/process_management/app_manager.c index 2427d617c6..60413ccd24 100644 --- a/src/fw/process_management/app_manager.c +++ b/src/fw/process_management/app_manager.c @@ -39,6 +39,9 @@ #ifdef CONFIG_SERVICE_MIC_CAPTURE #include "pbl/services/mic_capture/mic_capture_service.h" #endif +#ifdef CONFIG_SERVICE_AUDIO_ENCODER +#include "pbl/services/audio_encoder/audio_encoder.h" +#endif #endif #include "shell/normal/app_idle_timeout.h" #include "shell/normal/watchface.h" @@ -417,9 +420,12 @@ static void prv_app_cleanup(void) { vibe_pattern_clear_for_owner(VibePatternOwner_App); #ifndef CONFIG_RECOVERY_FW speaker_service_stop_for_task(PebbleTask_App); -#endif #ifdef CONFIG_SERVICE_MIC_CAPTURE mic_capture_service_stop_for_task(PebbleTask_App); +#endif +#ifdef CONFIG_SERVICE_AUDIO_ENCODER + audio_encoder_service_close_for_task(PebbleTask_App); +#endif #endif ble_app_cleanup(); diff --git a/src/fw/services/CMakeLists.txt b/src/fw/services/CMakeLists.txt index ba614554f9..a5678878e0 100644 --- a/src/fw/services/CMakeLists.txt +++ b/src/fw/services/CMakeLists.txt @@ -10,6 +10,7 @@ pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_CACHE app_cache) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_FETCH_ENDPOINT app_fetch_endpoint) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_GLANCES app_glances) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_PERMISSIONS app_permissions) +pbl_add_subdirectory_ifdef(CONFIG_SERVICE_AUDIO_ENCODER audio_encoder) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_MIC_MANAGER mic_manager) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_MIC_CAPTURE mic_capture) pbl_add_subdirectory_ifdef(CONFIG_SERVICE_APP_INBOX_SERVICE app_inbox_service) diff --git a/src/fw/services/Kconfig b/src/fw/services/Kconfig index a859713e8b..dcdbe435f7 100644 --- a/src/fw/services/Kconfig +++ b/src/fw/services/Kconfig @@ -12,6 +12,7 @@ rsource "app_cache/Kconfig" rsource "app_fetch_endpoint/Kconfig" rsource "app_glances/Kconfig" rsource "app_permissions/Kconfig" +rsource "audio_encoder/Kconfig" rsource "mic_manager/Kconfig" rsource "mic_capture/Kconfig" rsource "app_inbox_service/Kconfig" diff --git a/src/fw/services/audio_encoder/CMakeLists.txt b/src/fw/services/audio_encoder/CMakeLists.txt new file mode 100644 index 0000000000..a2f61e08fd --- /dev/null +++ b/src/fw/services/audio_encoder/CMakeLists.txt @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +set(sources + audio_encoder.c + audio_encoder_backends.c +) +if(CONFIG_SPEEX) + list(APPEND sources audio_encoder_speex.c) +endif() +pbl_library() +pbl_library_sources(${sources}) diff --git a/src/fw/services/audio_encoder/Kconfig b/src/fw/services/audio_encoder/Kconfig new file mode 100644 index 0000000000..6a2fc7a2a6 --- /dev/null +++ b/src/fw/services/audio_encoder/Kconfig @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2026 Core Devices LLC +# SPDX-License-Identifier: Apache-2.0 + +config SERVICE_AUDIO_ENCODER + bool "Audio encoder" + help + Codec-agnostic speech encoder shared by dictation and the app + Microphone API. Backends: Speex (CONFIG_SPEEX). + +if SERVICE_AUDIO_ENCODER + +module = SERVICE_AUDIO_ENCODER +module-str = Audio encoder +source "subsys/logging/Kconfig.template.log_level" + +endif diff --git a/src/fw/services/audio_encoder/audio_encoder.c b/src/fw/services/audio_encoder/audio_encoder.c new file mode 100644 index 0000000000..35ca29abcd --- /dev/null +++ b/src/fw/services/audio_encoder/audio_encoder.c @@ -0,0 +1,114 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "pbl/services/audio_encoder/audio_encoder.h" +#include "pbl/services/audio_encoder/audio_encoder_backend.h" + +#include "pbl/kernel/mutex.h" +#include + +PBL_LOG_MODULE_DEFINE(service_audio_encoder, CONFIG_SERVICE_AUDIO_ENCODER_LOG_LEVEL); + +static PBL_MUTEX_DEFINE(s_lock); + +typedef struct { + const AudioEncoderBackend *backend; //!< NULL when closed + PebbleTask owner; + AudioEncoderInfo info; +} AudioEncoderState; + +static AudioEncoderState s_state; + +void audio_encoder_service_init(void) { + s_state = (AudioEncoderState){}; +} + +static const AudioEncoderBackend *prv_find_backend(AudioCodec codec) { + size_t num_backends = 0; + const AudioEncoderBackend *const *backends = audio_encoder_get_backends(&num_backends); + for (size_t i = 0; i < num_backends; i++) { + if (backends[i]->codec == codec) { + return backends[i]; + } + } + return NULL; +} + +bool audio_encoder_service_is_codec_available(AudioCodec codec) { + return (prv_find_backend(codec) != NULL); +} + +//! Expects s_lock held +static void prv_close_locked(void) { + if (s_state.backend) { + s_state.backend->close(); + PBL_LOG_DBG("Encoder closed (owner %u)", s_state.owner); + } + s_state = (AudioEncoderState){}; +} + +bool audio_encoder_service_open(AudioCodec codec, PebbleTask owner, AudioEncoderInfo *info_out) { + const AudioEncoderBackend *backend = prv_find_backend(codec); + if (!backend || !info_out) { + return false; + } + + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (s_state.backend) { + if (owner != AUDIO_ENCODER_SYSTEM_OWNER) { + PBL_LOG_DBG("Encoder busy (owner %u), refusing %u", s_state.owner, owner); + pbl_mutex_unlock(&s_lock); + return false; + } + // The system takes precedence over an app + prv_close_locked(); + } + + AudioEncoderInfo info = {}; + if (!backend->open(&info)) { + PBL_LOG_ERR("Backend for codec %u failed to open", codec); + pbl_mutex_unlock(&s_lock); + return false; + } + info.codec = codec; + s_state = (AudioEncoderState){ + .backend = backend, + .owner = owner, + .info = info, + }; + *info_out = info; + pbl_mutex_unlock(&s_lock); + PBL_LOG_DBG("Encoder opened, codec %u, owner %u", codec, owner); + return true; +} + +int audio_encoder_service_encode(PebbleTask owner, const int16_t *pcm, uint32_t num_samples, + uint8_t *out, uint32_t out_len) { + if (!pcm || !out) { + return -1; + } + pbl_mutex_lock(&s_lock, PBL_FOREVER); + int rv = -1; + if (s_state.backend && (s_state.owner == owner) && + (num_samples == (uint32_t)s_state.info.frame_samples * s_state.info.channels)) { + rv = s_state.backend->encode(pcm, out, out_len); + } + pbl_mutex_unlock(&s_lock); + return rv; +} + +void audio_encoder_service_close(PebbleTask owner) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (s_state.backend && (s_state.owner == owner)) { + prv_close_locked(); + } + pbl_mutex_unlock(&s_lock); +} + +void audio_encoder_service_close_for_task(PebbleTask task) { + audio_encoder_service_close(task); +} + +bool audio_encoder_service_is_open(void) { + return (s_state.backend != NULL); +} diff --git a/src/fw/services/audio_encoder/audio_encoder_backends.c b/src/fw/services/audio_encoder/audio_encoder_backends.c new file mode 100644 index 0000000000..dc6c76d77d --- /dev/null +++ b/src/fw/services/audio_encoder/audio_encoder_backends.c @@ -0,0 +1,17 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "pbl/services/audio_encoder/audio_encoder_backend.h" + +#include "pbl/util/size.h" + +static const AudioEncoderBackend *const s_backends[] = { +#ifdef CONFIG_SPEEX + &g_audio_encoder_backend_speex, +#endif +}; + +const AudioEncoderBackend *const *audio_encoder_get_backends(size_t *num_backends_out) { + *num_backends_out = ARRAY_LENGTH(s_backends); + return s_backends; +} diff --git a/src/fw/services/audio_encoder/audio_encoder_speex.c b/src/fw/services/audio_encoder/audio_encoder_speex.c new file mode 100644 index 0000000000..c3b323b8a7 --- /dev/null +++ b/src/fw/services/audio_encoder/audio_encoder_speex.c @@ -0,0 +1,153 @@ +/* SPDX-FileCopyrightText: 2025 Joshua Jun */ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "pbl/services/audio_encoder/audio_encoder_backend.h" + +#include "board/board.h" +#include "kernel/pbl_malloc.h" +#include +#include + +#include "speex/speex.h" +#include "speex/speex_bits.h" +#include "speex/speex_stereo.h" + +#include +#include + +PBL_LOG_MODULE_DECLARE(service_audio_encoder, CONFIG_SERVICE_AUDIO_ENCODER_LOG_LEVEL); + +extern const SpeexMode speex_wb_mode; + +#define SPEEX_BITSTREAM_VERSION (4) +#define SPEEX_SAMPLE_RATE (16000) // 16 kHz wideband +#define SPEEX_BIT_RATE (9800) // 9.8 kbps +#define SPEEX_QUALITY (6) // Quality level (0-10) +#define SPEEX_COMPLEXITY (1) // Complexity (1-10, lower for embedded) +#define SPEEX_MAX_PACKET_BYTES (200) +#define SPEEX_AUDIO_GAIN (3) // Audio gain multiplier + +typedef struct { + void *enc_state; + SpeexBits bits; + SpeexStereoState stereo_state; + uint32_t frame_size; //!< samples per channel + uint8_t channels; + int16_t *work; //!< frame_size * channels samples; encoding modifies its input + bool open; +} SpeexEncoder; + +static SpeexEncoder s_encoder; + +static void prv_close(void) { + if (!s_encoder.open) { + return; + } + if (s_encoder.enc_state) { + speex_encoder_destroy(s_encoder.enc_state); + } + speex_bits_destroy(&s_encoder.bits); + kernel_free(s_encoder.work); + s_encoder = (SpeexEncoder){}; +} + +static bool prv_open(AudioEncoderInfo *info_out) { + if (s_encoder.open) { + return false; + } + s_encoder = (SpeexEncoder){ + .channels = (uint8_t)mic_get_channels(MIC), + }; + + s_encoder.enc_state = speex_encoder_init(&speex_wb_mode); + if (!s_encoder.enc_state) { + PBL_LOG_ERR("Failed to initialize Speex encoder"); + return false; + } + speex_bits_init(&s_encoder.bits); + if (s_encoder.channels == 2) { + s_encoder.stereo_state = (SpeexStereoState)SPEEX_STEREO_STATE_INIT; + } + + speex_encoder_ctl(s_encoder.enc_state, SPEEX_GET_FRAME_SIZE, &s_encoder.frame_size); + + int tmp = SPEEX_QUALITY; + speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_QUALITY, &tmp); + tmp = SPEEX_COMPLEXITY; + speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_COMPLEXITY, &tmp); + tmp = SPEEX_SAMPLE_RATE; + speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_SAMPLING_RATE, &tmp); + tmp = SPEEX_BIT_RATE; + speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_BITRATE, &tmp); + + int sample_rate = 0; + int bit_rate = 0; + speex_encoder_ctl(s_encoder.enc_state, SPEEX_GET_SAMPLING_RATE, &sample_rate); + speex_encoder_ctl(s_encoder.enc_state, SPEEX_GET_BITRATE, &bit_rate); + + s_encoder.work = kernel_malloc(s_encoder.frame_size * s_encoder.channels * sizeof(int16_t)); + if (!s_encoder.work) { + speex_encoder_destroy(s_encoder.enc_state); + speex_bits_destroy(&s_encoder.bits); + s_encoder = (SpeexEncoder){}; + return false; + } + s_encoder.open = true; + + *info_out = (AudioEncoderInfo){ + .codec = AudioCodecSpeexWB, + .channels = s_encoder.channels, + .frame_samples = (uint16_t)s_encoder.frame_size, + .sample_rate = (uint32_t)sample_rate, + .bitrate = (uint32_t)bit_rate, + .max_packet_bytes = SPEEX_MAX_PACKET_BYTES, + .bitstream_version = SPEEX_BITSTREAM_VERSION, + }; + PBL_LOG_DBG("Speex encoder opened: sample_rate=%d, bit_rate=%d, frame_size=%" PRIu32 + ", channels=%" PRIu8, + sample_rate, bit_rate, s_encoder.frame_size, s_encoder.channels); + if (sample_rate != MIC_SAMPLE_RATE) { + PBL_LOG_WRN("Speex sample rate (%d) != mic sample rate (%d)", sample_rate, MIC_SAMPLE_RATE); + } + return true; +} + +static int prv_encode(const int16_t *pcm, uint8_t *out, uint32_t out_len) { + if (!s_encoder.open) { + return -1; + } + const uint32_t total_samples = s_encoder.frame_size * s_encoder.channels; + + // Apply gain, clamped, into the work buffer + for (uint32_t i = 0; i < total_samples; i++) { + int32_t boosted = (int32_t)pcm[i] * SPEEX_AUDIO_GAIN; + if (boosted > INT16_MAX) { + boosted = INT16_MAX; + } else if (boosted < INT16_MIN) { + boosted = INT16_MIN; + } + s_encoder.work[i] = (int16_t)boosted; + } + + speex_bits_reset(&s_encoder.bits); + if (s_encoder.channels == 2) { + // Encodes the stereo info and folds the interleaved input to mono in place + speex_encode_stereo_int(s_encoder.work, s_encoder.frame_size, &s_encoder.bits); + } + speex_encode_int(s_encoder.enc_state, (spx_int16_t *)s_encoder.work, &s_encoder.bits); + + const int encoded_bytes = speex_bits_write(&s_encoder.bits, (char *)out, out_len); + if (encoded_bytes < 0) { + PBL_LOG_ERR("Failed to write Speex encoded data (%d)", encoded_bytes); + return -1; + } + return encoded_bytes; +} + +const AudioEncoderBackend g_audio_encoder_backend_speex = { + .codec = AudioCodecSpeexWB, + .open = prv_open, + .encode = prv_encode, + .close = prv_close, +}; diff --git a/src/fw/services/services_normal/service.c b/src/fw/services/services_normal/service.c index cc3f7ddc8e..6c48187ec0 100644 --- a/src/fw/services/services_normal/service.c +++ b/src/fw/services/services_normal/service.c @@ -14,6 +14,7 @@ #include "pbl/services/app_cache.h" #include "pbl/services/app_glances/app_glance_service.h" #include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/audio_encoder/audio_encoder.h" #include "pbl/services/mic_capture/mic_capture_service.h" #include "pbl/services/mic_manager.h" #include "pbl/services/blob_db/api.h" @@ -123,6 +124,9 @@ void services_normal_init(void) { speaker_service_init(); +#ifdef CONFIG_SERVICE_AUDIO_ENCODER + audio_encoder_service_init(); +#endif #ifdef CONFIG_SERVICE_MIC_MANAGER mic_manager_init(); #endif diff --git a/src/fw/services/voice/Kconfig b/src/fw/services/voice/Kconfig index e45b48289c..604ab38a86 100644 --- a/src/fw/services/voice/Kconfig +++ b/src/fw/services/voice/Kconfig @@ -5,6 +5,7 @@ config SERVICE_VOICE bool "Voice" default y if MIC select SPEEX + select SERVICE_AUDIO_ENCODER select SERVICE_MIC_MANAGER help Voice (dictation) service. Requires a microphone driver (CONFIG_MIC). diff --git a/src/fw/services/voice/voice_speex.c b/src/fw/services/voice/voice_speex.c index 194d7deaa2..10adbb621c 100644 --- a/src/fw/services/voice/voice_speex.c +++ b/src/fw/services/voice/voice_speex.c @@ -1,253 +1,96 @@ /* SPDX-FileCopyrightText: 2025 Joshua Jun */ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ /* SPDX-License-Identifier: Apache-2.0 */ #include "pbl/services/voice/voice_speex.h" -#include "board/board.h" -#include -#include "system/passert.h" #include "kernel/pbl_malloc.h" +#include "pbl/services/audio_encoder/audio_encoder.h" +#include "system/passert.h" #include -#include - -#include "speex/speex.h" -#include "speex/speex_bits.h" -#include "speex/speex_header.h" -#include "speex/speex_stereo.h" -#include #include +#include PBL_LOG_MODULE_DECLARE(service_voice, CONFIG_SERVICE_VOICE_LOG_LEVEL); -// External Speex mode declarations -extern const SpeexMode speex_wb_mode; - -// Speex bitstream version -#define SPEEX_BITSTREAM_VERSION 4 - -// Speex encoder state +// Dictation's view of the shared audio encoder: the Speex backend opened as the system owner, +// plus the buffer the mic driver fills with one frame at a time. typedef struct { - void *enc_state; - SpeexBits bits; - SpeexStereoState stereo_state; - uint32_t frame_size; // Changed to uint32_t to match transfer info - uint32_t sample_rate; // Changed to uint32_t to match transfer info - uint16_t bit_rate; // Changed to uint16_t to match transfer info - uint8_t bitstream_version; // Changed to uint8_t to match transfer info - uint8_t channels; // 1 for mono, 2 for stereo + AudioEncoderInfo info; + int16_t *frame_buffer; bool initialized; - uint8_t *frame_buffer; - size_t frame_buffer_size; - uint8_t *encoded_buffer; - size_t encoded_buffer_size; -} VoiceSpeexEncoder; - -static VoiceSpeexEncoder s_encoder = {0}; - -// Speex configuration -#define SPEEX_SAMPLE_RATE 16000 // 16 kHz wideband -#define SPEEX_BIT_RATE 9800 // 9.8 kbps -#define SPEEX_QUALITY 6 // Quality level (0-10) -#define SPEEX_COMPLEXITY 1 // Complexity (1-10, lower for embedded) -#define SPEEX_ENCODED_BUFFER_SIZE 320 // Max encoded frame size -#define SPEEX_AUDIO_GAIN 3 // Audio gain multiplier (3x) +} VoiceSpeexState; + +static VoiceSpeexState s_speex; + +static size_t prv_frame_buffer_size(void) { + return (size_t)s_speex.info.frame_samples * s_speex.info.channels * sizeof(int16_t); +} bool voice_speex_init(void) { - if (s_encoder.initialized) { + if (s_speex.initialized) { return true; } - - memset(&s_encoder, 0, sizeof(s_encoder)); - - // Get channel count from mic device (default to mono if not specified) - s_encoder.channels = (uint8_t)mic_get_channels(MIC); - PBL_LOG_DBG("Mic channels: %" PRIu8, s_encoder.channels); - - // Initialize Speex encoder - use wideband mode for 16kHz sample rate - const SpeexMode *mode = &speex_wb_mode; - if (!mode) { - PBL_LOG_ERR("Failed to get Speex wideband mode"); + if (!audio_encoder_service_open(AudioCodecSpeexWB, AUDIO_ENCODER_SYSTEM_OWNER, &s_speex.info)) { + PBL_LOG_ERR("Failed to open Speex encoder"); return false; } - - s_encoder.enc_state = speex_encoder_init(mode); - if (!s_encoder.enc_state) { - PBL_LOG_ERR("Failed to initialize Speex encoder"); + s_speex.frame_buffer = kernel_malloc(prv_frame_buffer_size()); + if (!s_speex.frame_buffer) { + audio_encoder_service_close(AUDIO_ENCODER_SYSTEM_OWNER); return false; } - - // Initialize bits structure - speex_bits_init(&s_encoder.bits); - - // Initialize stereo state if stereo - if (s_encoder.channels == 2) { - s_encoder.stereo_state = (SpeexStereoState)SPEEX_STEREO_STATE_INIT; - } - - // Get frame size - speex_encoder_ctl(s_encoder.enc_state, SPEEX_GET_FRAME_SIZE, &s_encoder.frame_size); - PBL_LOG_DBG("Initial frame size from Speex: %" PRIu32, s_encoder.frame_size); - - // Set encoder parameters - int tmp = SPEEX_QUALITY; - speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_QUALITY, &tmp); - - tmp = SPEEX_COMPLEXITY; - speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_COMPLEXITY, &tmp); - - tmp = SPEEX_SAMPLE_RATE; - speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_SAMPLING_RATE, &tmp); - PBL_LOG_DBG("Set sample rate to: %d", tmp); - - tmp = SPEEX_BIT_RATE; - speex_encoder_ctl(s_encoder.enc_state, SPEEX_SET_BITRATE, &tmp); - PBL_LOG_DBG("Set bit rate to: %d", tmp); - - // Get actual parameters - int actual_sample_rate, actual_bit_rate; - speex_encoder_ctl(s_encoder.enc_state, SPEEX_GET_SAMPLING_RATE, &actual_sample_rate); - speex_encoder_ctl(s_encoder.enc_state, SPEEX_GET_BITRATE, &actual_bit_rate); - - s_encoder.sample_rate = (uint32_t)actual_sample_rate; - s_encoder.bit_rate = (uint16_t)actual_bit_rate; - - s_encoder.bitstream_version = SPEEX_BITSTREAM_VERSION; - - // Allocate frame buffer (16-bit samples, multiplied by channel count for stereo) - s_encoder.frame_buffer_size = s_encoder.frame_size * sizeof(int16_t) * s_encoder.channels; - s_encoder.frame_buffer = (uint8_t *)kernel_malloc_check(s_encoder.frame_buffer_size); - - // Allocate encoded buffer - s_encoder.encoded_buffer_size = SPEEX_ENCODED_BUFFER_SIZE; - s_encoder.encoded_buffer = kernel_malloc_check(s_encoder.encoded_buffer_size); - - s_encoder.initialized = true; - - PBL_LOG_DBG("Speex encoder initialized: sample_rate=%" PRIu32 ", bit_rate=%" PRIu16 - ", frame_size=%" PRIu32 ", channels=%" PRIu8, - s_encoder.sample_rate, s_encoder.bit_rate, s_encoder.frame_size, s_encoder.channels); - - // Verify sample rates match - if (s_encoder.sample_rate != MIC_SAMPLE_RATE) { - PBL_LOG_WRN("Speex sample rate (%" PRIu32 ") != Mic sample rate (%d)", s_encoder.sample_rate, - MIC_SAMPLE_RATE); - } - + s_speex.initialized = true; return true; } void voice_speex_deinit(void) { - if (!s_encoder.initialized) { + if (!s_speex.initialized) { return; } - - if (s_encoder.enc_state) { - speex_encoder_destroy(s_encoder.enc_state); - s_encoder.enc_state = NULL; - } - - speex_bits_destroy(&s_encoder.bits); - - if (s_encoder.frame_buffer) { - kernel_free(s_encoder.frame_buffer); - s_encoder.frame_buffer = NULL; - } - - if (s_encoder.encoded_buffer) { - kernel_free(s_encoder.encoded_buffer); - s_encoder.encoded_buffer = NULL; - } - - memset(&s_encoder, 0, sizeof(s_encoder)); + audio_encoder_service_close(AUDIO_ENCODER_SYSTEM_OWNER); + kernel_free(s_speex.frame_buffer); + s_speex = (VoiceSpeexState){}; } void voice_speex_get_transfer_info(AudioTransferInfoSpeex *info) { - PBL_ASSERTN(s_encoder.initialized); + PBL_ASSERTN(s_speex.initialized); PBL_ASSERTN(info); memset(info, 0, sizeof(AudioTransferInfoSpeex)); strncpy(info->version, "1.2.1", sizeof(info->version) - 1); - info->sample_rate = s_encoder.sample_rate; - info->bit_rate = s_encoder.bit_rate; - info->frame_size = (uint16_t)s_encoder.frame_size; // Explicit cast to uint16_t - info->bitstream_version = s_encoder.bitstream_version; + info->sample_rate = s_speex.info.sample_rate; + info->bit_rate = (uint16_t)s_speex.info.bitrate; + info->frame_size = s_speex.info.frame_samples; + info->bitstream_version = s_speex.info.bitstream_version; PBL_LOG_DBG("Transfer info: sample_rate=%" PRIu32 ", bit_rate=%" PRIu16 ", frame_size=%" PRIu16 ", bitstream_version=%" PRIu8, info->sample_rate, info->bit_rate, info->frame_size, info->bitstream_version); - - // Additional validation - if (info->sample_rate != 16000) { - PBL_LOG_WRN("Unexpected sample rate in transfer info: %" PRIu32, info->sample_rate); - } } int voice_speex_get_frame_size(void) { - // Return total samples per frame (frame_size * channels for stereo) - return s_encoder.initialized ? (int)(s_encoder.frame_size * s_encoder.channels) : 0; + return s_speex.initialized ? (int)(s_speex.info.frame_samples * s_speex.info.channels) : 0; } int16_t *voice_speex_get_frame_buffer(void) { - return s_encoder.initialized ? (int16_t *)s_encoder.frame_buffer : NULL; + return s_speex.initialized ? s_speex.frame_buffer : NULL; } size_t voice_speex_get_frame_buffer_size(void) { - return s_encoder.initialized ? s_encoder.frame_buffer_size : 0; + return s_speex.initialized ? prv_frame_buffer_size() : 0; } int voice_speex_encode_frame(int16_t *samples, uint8_t *encoded_data, size_t max_encoded_size) { - if (!s_encoder.initialized) { + if (!s_speex.initialized) { PBL_LOG_ERR("encode_frame called but Speex not initialized"); return -1; } - - if (!samples || !encoded_data) { - PBL_LOG_ERR("encode_frame called with invalid buffers"); - return -1; - } - - uint32_t total_samples = s_encoder.frame_size * s_encoder.channels; - - // Apply gain boost to samples - for (uint32_t i = 0; i < total_samples; i++) { - int32_t boosted = (int32_t)samples[i] * SPEEX_AUDIO_GAIN; - // Clamp to int16_t range to prevent overflow - if (boosted > INT16_MAX) { - boosted = INT16_MAX; - } else if (boosted < INT16_MIN) { - boosted = INT16_MIN; - } - samples[i] = (int16_t)boosted; - } - - // Reset bits structure - speex_bits_reset(&s_encoder.bits); - - if (s_encoder.channels == 2) { - // For stereo: encode stereo info and convert to mono in-place - speex_encode_stereo_int(samples, s_encoder.frame_size, &s_encoder.bits); - } - - // Encode frame (for stereo, samples have been converted to mono in-place by - // speex_encode_stereo_int) - speex_encode_int(s_encoder.enc_state, (spx_int16_t *)samples, &s_encoder.bits); - - // Write encoded data to buffer - int encoded_bytes = speex_bits_write(&s_encoder.bits, (char *)encoded_data, max_encoded_size); - - if (encoded_bytes < 0) { - PBL_LOG_ERR("Failed to write Speex encoded data (returned %d)", encoded_bytes); - return -1; - } - - PBL_LOG_VERBOSE("Encoded frame: input_samples=%" PRIu32 ", output_bytes=%d, frame_size=%" PRIu32 - ", channels=%" PRIu8, - total_samples, encoded_bytes, s_encoder.frame_size, s_encoder.channels); - - return encoded_bytes; + return audio_encoder_service_encode(AUDIO_ENCODER_SYSTEM_OWNER, samples, + voice_speex_get_frame_size(), encoded_data, max_encoded_size); } bool voice_speex_is_initialized(void) { - return s_encoder.initialized; + return s_speex.initialized; } diff --git a/tests/fw/services/CMakeLists.txt b/tests/fw/services/CMakeLists.txt index 79dfb57440..4b790e2c28 100644 --- a/tests/fw/services/CMakeLists.txt +++ b/tests/fw/services/CMakeLists.txt @@ -305,6 +305,12 @@ pbl_clar_test(test_app_cache OVERRIDES dummy_board ) +pbl_clar_test(test_audio_encoder + SOURCES + src/fw/services/audio_encoder/audio_encoder.c + OVERRIDES dummy_board +) + pbl_clar_test(test_mic_manager SOURCES src/fw/services/mic_manager/mic_manager.c diff --git a/tests/fw/services/test_audio_encoder.c b/tests/fw/services/test_audio_encoder.c new file mode 100644 index 0000000000..167f1da5a0 --- /dev/null +++ b/tests/fw/services/test_audio_encoder.c @@ -0,0 +1,178 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "clar.h" + +#include "pbl/services/audio_encoder/audio_encoder.h" +#include "pbl/services/audio_encoder/audio_encoder_backend.h" + +#include + +// Fakes +//////////////////////////////////////////////////////////////// +#include "fake_mutex.h" + +// Stubs +//////////////////////////////////////////////////////////////// +#include "stubs_logging.h" +#include "stubs_passert.h" + +// Fake backend +//////////////////////////////////////////////////////////////// + +static int s_open_count; +static int s_close_count; +static int s_encode_count; +static bool s_open_should_fail; +static int16_t s_last_first_sample; + +#define FAKE_FRAME_SAMPLES (160) + +static bool prv_open(AudioEncoderInfo *info_out) { + if (s_open_should_fail) { + return false; + } + s_open_count++; + *info_out = (AudioEncoderInfo){ + .channels = 1, + .frame_samples = FAKE_FRAME_SAMPLES, + .sample_rate = 16000, + .bitrate = 12000, + .max_packet_bytes = 40, + .bitstream_version = 1, + }; + return true; +} + +static int prv_encode(const int16_t *pcm, uint8_t *out, uint32_t out_len) { + s_encode_count++; + s_last_first_sample = pcm[0]; + memset(out, 0xAB, out_len < 30 ? out_len : 30); + return 30; +} + +static void prv_close(void) { + s_close_count++; +} + +static const AudioEncoderBackend s_fake_backend = { + .codec = AudioCodecSpeexWB, + .open = prv_open, + .encode = prv_encode, + .close = prv_close, +}; + +static const AudioEncoderBackend *const s_backends[] = {&s_fake_backend}; + +const AudioEncoderBackend *const *audio_encoder_get_backends(size_t *num_backends_out) { + *num_backends_out = 1; + return s_backends; +} + +// Setup +//////////////////////////////////////////////////////////////// + +void test_audio_encoder__initialize(void) { + fake_mutex_reset(false); + s_open_count = 0; + s_close_count = 0; + s_encode_count = 0; + s_open_should_fail = false; + audio_encoder_service_init(); +} + +void test_audio_encoder__cleanup(void) { + fake_mutex_assert_all_unlocked(); +} + +// Tests +//////////////////////////////////////////////////////////////// + +void test_audio_encoder__codec_availability(void) { + cl_assert(audio_encoder_service_is_codec_available(AudioCodecSpeexWB)); + cl_assert(!audio_encoder_service_is_codec_available(AudioCodecInvalid)); + cl_assert(!audio_encoder_service_is_codec_available((AudioCodec)7)); + cl_assert(!audio_encoder_service_is_codec_available(AudioCodecCount)); +} + +void test_audio_encoder__open_encode_close(void) { + AudioEncoderInfo info; + cl_assert(!audio_encoder_service_open((AudioCodec)7, PebbleTask_App, &info)); + cl_assert(!audio_encoder_service_open(AudioCodecSpeexWB, PebbleTask_App, NULL)); + cl_assert(!audio_encoder_service_is_open()); + + cl_assert(audio_encoder_service_open(AudioCodecSpeexWB, PebbleTask_App, &info)); + cl_assert(audio_encoder_service_is_open()); + cl_assert_equal_i(1, s_open_count); + cl_assert_equal_i(info.codec, AudioCodecSpeexWB); + cl_assert_equal_i(info.frame_samples, FAKE_FRAME_SAMPLES); + cl_assert_equal_i(info.sample_rate, 16000); + + int16_t pcm[FAKE_FRAME_SAMPLES] = {[0] = 1234}; + uint8_t out[64]; + // Wrong owner + cl_assert(audio_encoder_service_encode(PebbleTask_Worker, pcm, FAKE_FRAME_SAMPLES, out, + sizeof(out)) < 0); + // Wrong frame size + cl_assert(audio_encoder_service_encode(PebbleTask_App, pcm, FAKE_FRAME_SAMPLES - 1, out, + sizeof(out)) < 0); + cl_assert( + audio_encoder_service_encode(PebbleTask_App, NULL, FAKE_FRAME_SAMPLES, out, sizeof(out)) < 0); + cl_assert_equal_i(0, s_encode_count); + + cl_assert_equal_i( + 30, audio_encoder_service_encode(PebbleTask_App, pcm, FAKE_FRAME_SAMPLES, out, sizeof(out))); + cl_assert_equal_i(1, s_encode_count); + cl_assert_equal_i(1234, s_last_first_sample); + cl_assert_equal_i(0xAB, out[0]); + + // Close by non-owner is ignored + audio_encoder_service_close(PebbleTask_Worker); + cl_assert(audio_encoder_service_is_open()); + audio_encoder_service_close(PebbleTask_App); + cl_assert(!audio_encoder_service_is_open()); + cl_assert_equal_i(1, s_close_count); + cl_assert( + audio_encoder_service_encode(PebbleTask_App, pcm, FAKE_FRAME_SAMPLES, out, sizeof(out)) < 0); + + // Double close is harmless + audio_encoder_service_close(PebbleTask_App); + cl_assert_equal_i(1, s_close_count); +} + +void test_audio_encoder__backend_open_failure(void) { + AudioEncoderInfo info; + s_open_should_fail = true; + cl_assert(!audio_encoder_service_open(AudioCodecSpeexWB, PebbleTask_App, &info)); + cl_assert(!audio_encoder_service_is_open()); +} + +void test_audio_encoder__system_preempts_app_but_not_vice_versa(void) { + AudioEncoderInfo info; + cl_assert(audio_encoder_service_open(AudioCodecSpeexWB, PebbleTask_App, &info)); + + // Another app-level open is refused + cl_assert(!audio_encoder_service_open(AudioCodecSpeexWB, PebbleTask_Worker, &info)); + cl_assert_equal_i(1, s_open_count); + + // The system takes it over + cl_assert(audio_encoder_service_open(AudioCodecSpeexWB, AUDIO_ENCODER_SYSTEM_OWNER, &info)); + cl_assert_equal_i(2, s_open_count); + cl_assert_equal_i(1, s_close_count); + + int16_t pcm[FAKE_FRAME_SAMPLES] = {}; + uint8_t out[64]; + cl_assert( + audio_encoder_service_encode(PebbleTask_App, pcm, FAKE_FRAME_SAMPLES, out, sizeof(out)) < 0); + cl_assert_equal_i(30, audio_encoder_service_encode(AUDIO_ENCODER_SYSTEM_OWNER, pcm, + FAKE_FRAME_SAMPLES, out, sizeof(out))); + + // The app cannot take it back while the system holds it + cl_assert(!audio_encoder_service_open(AudioCodecSpeexWB, PebbleTask_App, &info)); + + audio_encoder_service_close_for_task(PebbleTask_App); // no-op + cl_assert(audio_encoder_service_is_open()); + audio_encoder_service_close(AUDIO_ENCODER_SYSTEM_OWNER); + cl_assert(!audio_encoder_service_is_open()); + cl_assert_equal_i(2, s_close_count); +} From 3dd8ff35dc3f8a7e15584276aa006ef4df41f436 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 17:59:04 +0200 Subject: [PATCH 12/15] applib: add AudioEncoder API Lets an app compress microphone frames with the firmware's speech encoder (Speex wideband today) before shipping them to the phone over AppMessage: audio_encoder_open() returns the frame size and packet bound, audio_encoder_encode_frame() produces one packet per frame. Exported as SDK revision 112. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- src/fw/applib/audio_encoder.c | 29 +++++++++ src/fw/applib/audio_encoder.h | 59 ++++++++++++++++++ .../process_management/pebble_process_info.h | 3 +- src/fw/syscall/syscall.h | 8 +++ src/fw/syscall/syscall_audio_encoder.c | 62 +++++++++++++++++++ .../generate_native_sdk/exported_symbols.json | 36 ++++++++++- 6 files changed, 194 insertions(+), 3 deletions(-) create mode 100644 src/fw/applib/audio_encoder.c create mode 100644 src/fw/applib/audio_encoder.h create mode 100644 src/fw/syscall/syscall_audio_encoder.c diff --git a/src/fw/applib/audio_encoder.c b/src/fw/applib/audio_encoder.c new file mode 100644 index 0000000000..ca221026cd --- /dev/null +++ b/src/fw/applib/audio_encoder.c @@ -0,0 +1,29 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "applib/audio_encoder.h" + +#include "syscall/syscall.h" + +bool audio_encoder_codec_available(AudioCodec codec) { + return sys_audio_encoder_codec_available((uint8_t)codec); +} + +bool audio_encoder_open(AudioCodec codec, AudioEncoderInfo *info_out) { + if (!info_out) { + return false; + } + return sys_audio_encoder_open((uint8_t)codec, info_out); +} + +int audio_encoder_encode_frame(const int16_t *pcm, uint32_t num_samples, uint8_t *out, + uint32_t out_len) { + if (!pcm || !out) { + return -1; + } + return sys_audio_encoder_encode(pcm, num_samples, out, out_len); +} + +void audio_encoder_close(void) { + sys_audio_encoder_close(); +} diff --git a/src/fw/applib/audio_encoder.h b/src/fw/applib/audio_encoder.h new file mode 100644 index 0000000000..892a214db1 --- /dev/null +++ b/src/fw/applib/audio_encoder.h @@ -0,0 +1,59 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#pragma once + +#include "pbl/services/audio_encoder/audio_encoder_types.h" + +#include +#include + +//! @addtogroup Foundation +//! @{ +//! @addtogroup AudioEncoder +//! \brief Compressing microphone audio before sending it to the phone +//! +//! Raw 16 kHz PCM is far too much for the Bluetooth link, so an app that streams audio to its +//! phone-side component should compress it first. The firmware provides a speech encoder the app +//! can feed one frame at a time; the resulting packets are small enough to batch into +//! AppMessages. A typical loop: +//! +//! \code{.c} +//! AudioEncoderInfo info; +//! audio_encoder_open(AudioCodecSpeexWB, &info); +//! mic_data_service_subscribe(info.frame_samples, handlers, NULL); +//! // in the data handler: +//! uint8_t packet[info.max_packet_bytes]; +//! int len = audio_encoder_encode_frame(samples, num_samples, packet, sizeof(packet)); +//! // append packet to an AppMessage buffer and send every N frames +//! \endcode +//! +//! Send the \ref AudioEncoderInfo to the phone once so it can configure its decoder. Only one +//! encoder can be open at a time and the system may take it away for dictation, in which case +//! capture stops first (see \ref MicDataStopReasonInterrupted) and \ref audio_encoder_encode_frame +//! fails until the app opens it again. +//! @{ + +//! @return true if this firmware can encode with the codec +bool audio_encoder_codec_available(AudioCodec codec); + +//! Opens the encoder. +//! @param codec Codec to encode with +//! @param[out] info_out Frame size, rates and packet bound of the opened encoder +//! @return false if the codec is unavailable or the encoder is in use +bool audio_encoder_open(AudioCodec codec, AudioEncoderInfo *info_out); + +//! Encodes one frame. +//! @param pcm Exactly `frame_samples * channels` samples from \ref AudioEncoderInfo +//! @param num_samples Number of samples in pcm +//! @param out Buffer of at least `max_packet_bytes` +//! @param out_len Size of out +//! @return Number of bytes written to out, or a negative value on error +int audio_encoder_encode_frame(const int16_t *pcm, uint32_t num_samples, uint8_t *out, + uint32_t out_len); + +//! Closes the encoder. Safe to call when not open. +void audio_encoder_close(void); + +//! @} // end addtogroup AudioEncoder +//! @} // end addtogroup Foundation diff --git a/src/fw/process_management/pebble_process_info.h b/src/fw/process_management/pebble_process_info.h index 33b43d14db..cb986de201 100644 --- a/src/fw/process_management/pebble_process_info.h +++ b/src/fw/process_management/pebble_process_info.h @@ -172,9 +172,10 @@ typedef enum { // .minor:0x6a -- Add HRV sampling API (health_service_set_hrv_sample_period) (rev 109) // sdk.major:0x5 .minor:0x6b -- Add app permissions API (rev 110) // sdk.major:0x5 .minor:0x6c -- Add Microphone API (mic_data_service_subscribe) (rev 111) +// sdk.major:0x5 .minor:0x6d -- Add AudioEncoder API (audio_encoder_open) (rev 112) #define PROCESS_INFO_CURRENT_SDK_VERSION_MAJOR 0x5 -#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6c +#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6d // The first SDK to ship with 2.x APIs #define PROCESS_INFO_FIRST_2X_SDK_VERSION_MAJOR 0x4 diff --git a/src/fw/syscall/syscall.h b/src/fw/syscall/syscall.h index 84582cc72e..327c62b082 100644 --- a/src/fw/syscall/syscall.h +++ b/src/fw/syscall/syscall.h @@ -4,6 +4,7 @@ #pragma once #include "applib/app_comm.h" +#include "pbl/services/audio_encoder/audio_encoder_types.h" #include "applib/app_exit_reason.h" #include "applib/app_inbox.h" #include "applib/app_outbox.h" @@ -113,6 +114,13 @@ uint32_t sys_mic_capture_read(int16_t *out, uint32_t max_samples); uint32_t sys_mic_capture_get_available(void); bool sys_mic_capture_is_active(void); +//! App-owned speech encoder. See audio_encoder.h. +bool sys_audio_encoder_codec_available(uint8_t codec); +bool sys_audio_encoder_open(uint8_t codec, AudioEncoderInfo *info_out); +int sys_audio_encoder_encode(const int16_t *pcm, uint32_t num_samples, uint8_t *out, + uint32_t out_len); +void sys_audio_encoder_close(void); + void sys_get_app_uuid(Uuid *uuid); bool sys_app_is_watchface(void); AppInstallId sys_app_manager_get_current_app_id(void); diff --git a/src/fw/syscall/syscall_audio_encoder.c b/src/fw/syscall/syscall_audio_encoder.c new file mode 100644 index 0000000000..4a808fea17 --- /dev/null +++ b/src/fw/syscall/syscall_audio_encoder.c @@ -0,0 +1,62 @@ +/* SPDX-FileCopyrightText: 2026 Core Devices LLC */ +/* SPDX-License-Identifier: Apache-2.0 */ + +#include "syscall/syscall.h" +#include "syscall/syscall_internal.h" + +#include "kernel/pebble_tasks.h" +#include "pbl/services/audio_encoder/audio_encoder.h" + +// Builds without the encoder service (e.g. PRF) report no codec and refuse every request. + +DEFINE_SYSCALL(bool, sys_audio_encoder_codec_available, uint8_t codec) { +#ifdef CONFIG_SERVICE_AUDIO_ENCODER + return audio_encoder_service_is_codec_available((AudioCodec)codec); +#else + return false; +#endif +} + +DEFINE_SYSCALL(bool, sys_audio_encoder_open, uint8_t codec, AudioEncoderInfo *info_out) { + if (PRIVILEGE_WAS_ELEVATED) { + syscall_assert_userspace_buffer(info_out, sizeof(*info_out)); + } + // Only the app task may hold the app encoder; the system owner is reserved for dictation. + const PebbleTask task = pebble_task_get_current(); + if (task != PebbleTask_App) { + return false; + } +#ifdef CONFIG_SERVICE_AUDIO_ENCODER + AudioEncoderInfo info; + if (!audio_encoder_service_open((AudioCodec)codec, task, &info)) { + return false; + } + *info_out = info; + return true; +#else + return false; +#endif +} + +DEFINE_SYSCALL(int, sys_audio_encoder_encode, const int16_t *pcm, uint32_t num_samples, + uint8_t *out, uint32_t out_len) { + if (PRIVILEGE_WAS_ELEVATED) { + if ((num_samples > AUDIO_ENCODER_MAX_FRAME_SAMPLES) || + (out_len > AUDIO_ENCODER_MAX_PACKET_BYTES)) { + syscall_failed(); + } + syscall_assert_userspace_buffer(pcm, num_samples * sizeof(int16_t)); + syscall_assert_userspace_buffer(out, out_len); + } +#ifdef CONFIG_SERVICE_AUDIO_ENCODER + return audio_encoder_service_encode(pebble_task_get_current(), pcm, num_samples, out, out_len); +#else + return -1; +#endif +} + +DEFINE_SYSCALL(void, sys_audio_encoder_close, void) { +#ifdef CONFIG_SERVICE_AUDIO_ENCODER + audio_encoder_service_close(pebble_task_get_current()); +#endif +} diff --git a/tools/generate_native_sdk/exported_symbols.json b/tools/generate_native_sdk/exported_symbols.json index 7d24d27aba..521cc79612 100644 --- a/tools/generate_native_sdk/exported_symbols.json +++ b/tools/generate_native_sdk/exported_symbols.json @@ -4,7 +4,7 @@ "You should also make sure you are obeying our API design guidelines:", "https://pebbletechnology.atlassian.net/wiki/display/DEV/SDK+API+Design+Guidelines" ], - "revision" : "111", + "revision" : "112", "version" : "2.0", "files": [ "include/pbl/drivers/ambient_light.h", @@ -138,7 +138,9 @@ "fw/applib/unobstructed_area_service.h", "include/pbl/services/app_permissions/app_permissions_types.h", "fw/applib/app_permissions.h", - "fw/applib/mic_data_service.h" + "fw/applib/mic_data_service.h", + "include/pbl/services/audio_encoder/audio_encoder_types.h", + "fw/applib/audio_encoder.h" ], "exports": [ { @@ -4826,6 +4828,36 @@ "addedRevision": "111" } ] + }, { + "type": "group", + "name": "AudioEncoder", + "appOnly": true, + "addedRevision": "112", + "exports": [ + { + "type": "type", + "name": "AudioCodec" + }, { + "type": "type", + "name": "AudioEncoderInfo" + }, { + "type": "function", + "name": "audio_encoder_codec_available", + "addedRevision": "112" + }, { + "type": "function", + "name": "audio_encoder_open", + "addedRevision": "112" + }, { + "type": "function", + "name": "audio_encoder_encode_frame", + "addedRevision": "112" + }, { + "type": "function", + "name": "audio_encoder_close", + "addedRevision": "112" + } + ] }, { "type": "group", "name": "Light", From 7329ce28a9167f4921d432cc92a869701c1c08d3 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 18:12:23 +0200 Subject: [PATCH 13/15] docs: describe app permissions and the Microphone API Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- docs/development/app_permissions.md | 116 ++++++++++++++++++++++++++++ docs/index.md | 1 + 2 files changed, 117 insertions(+) create mode 100644 docs/development/app_permissions.md diff --git a/docs/development/app_permissions.md b/docs/development/app_permissions.md new file mode 100644 index 0000000000..ea72195150 --- /dev/null +++ b/docs/development/app_permissions.md @@ -0,0 +1,116 @@ +# App permissions and the Microphone API + +Third-party apps can ask for capabilities the user has to grant. The first +one is live microphone access. This page describes the moving parts across +the firmware and the phone app so the two stay in sync. + +Nothing that predates the permission system is gated by it: dictation +sessions keep working for every app, because the app only ever receives a +transcription. Permissions only guard APIs that hand raw data to the app. + +## Declaring a permission + +An app declares what it needs in its manifest: + +```json +"capabilities": ["microphone"] +``` + +The SDK build turns that into `PROCESS_INFO_USES_MICROPHONE` (bit 11 of the +binary header flags, see `pebble_process_info.h`), so the firmware can tell +"not declared" from "denied" without talking to the phone. The phone reads +the same array from `appinfo.json` in the `.pbw` and from the web locker. + +## Grant records + +The phone is the source of truth. It pushes one record per app into the +`AppPermissions` BlobDB (id `0x0D`), keyed by the 16-byte app UUID: + +| Offset | Size | Field | Notes | +|-------:|-----:|-----------------|------------------------------------| +| 0 | 1 | `version` | 1 | +| 1 | 3 | reserved | 0 | +| 4 | 4 | `granted_mask` | little-endian, bit 0 = microphone | +| 8 | 4 | `declared_mask` | informational, same bit layout | + +The record is deleted when the app is uninstalled and kept across upgrades, +so grants persist across ordinary updates. The watch advertises +`app_permissions_support` (protocol capability bit 25) so the phone knows it +can push records; enforcement never depends on that bit. + +## Resolving a permission on the watch + +`app_permissions_get_state_for_current_app()` in +`src/fw/services/app_permissions/` combines the header flag and the record: + +- not declared in the header → `AppPermissionStateNotDeclared`, whatever the + record says; +- system apps → `AppPermissionStateGranted`; +- declared, no record (old phone app, or nothing pushed yet) → + `AppPermissionStateDenied`. SDK shell builds (`CONFIG_SHELL_SDK`) return + `Granted` here so `pbl install` works without the phone; +- declared with a record → the bit in `granted_mask`. + +When the running app's record changes the service emits +`PEBBLE_APP_PERMISSION_EVENT`, which apps observe through +`app_permission_service_subscribe()`, and tells the capture service to stop +if the microphone was revoked. + +Console commands for development: + +``` +perm list +perm grant mic +perm revoke mic +``` + +## Microphone capture + +Watchfaces can never record: the SDK build rejects `microphone` in a +watchface manifest, the phone never creates a grant for one, and the capture +service refuses them (`MicCaptureStartErrWatchface`) regardless of the record. + +`mic_data_service_subscribe()` (applib) starts capture through +`mic_capture_service` (kernel). Capture is only served to the app task, only +while the app is in focus, and only with the permission granted. The kernel +owns a 320 ms ring buffer that the app drains through syscalls; when the +app falls behind the newest chunk is dropped and the next batch carries an +`overrun` flag. + +The microphone itself is shared through `mic_manager`: dictation always +wins and preempts an app, and an app cannot start while dictation runs. + +Capture stops, with a reason delivered to the app's `stopped` handler: + +| Trigger | `MicDataStopReason` | +|-------------------------------------------|----------------------| +| any focusable modal (notification, call…) | `FocusLost` | +| dictation takes the microphone | `Interrupted` | +| the grant is revoked | `PermissionRevoked` | +| the app exits or is killed | (no callback) | + +While capturing, the OS shows a "Listening" banner (`src/fw/popups/mic_banner.c`) +as a transparent, unfocusable modal at discreet priority. It reserves its strip +through the unobstructed area service, composing with Timeline Peek, so +`layer_get_unobstructed_bounds()` shrinks for the app. Any modal that could +hide the banner also takes focus, which is what stops capture. + +## Encoding + +Raw PCM is too much for the Bluetooth link. `audio_encoder_open()` (applib) +gives the app the firmware's speech encoder, one frame at a time; the +returned `AudioEncoderInfo` tells the phone how to decode. Speex wideband is +the only codec today; the service is codec-agnostic so another backend can +be added later. Dictation uses the same service as the system owner and +takes precedence over an app. + +## Phone side (CoreApp) + +- `LockerAppPermission` rows track each declared permission per app, with + `decided = false` until the user answers the prompt shown on the home + screen. Grants can be changed later from the app's detail screen. +- `LockerPermissions` reconciles the rows whenever an app is installed, + updated or removed, and mirrors them into the `AppPermissionsEntry` + BlobDB entity that syncs to the watch. +- An update that adds a permission installs with the permission denied and + prompts right away; the install itself is never blocked. diff --git a/docs/index.md b/docs/index.md index 3b8480e99f..ce9082d6ec 100644 --- a/docs/index.md +++ b/docs/index.md @@ -94,6 +94,7 @@ development/qemu.md development/debugging.md development/moddable.md development/sdk_export.md +development/app_permissions.md development/contributing.md ``` From 7611e3c0cbf73aa7d16c909e367d081248e6b6dc Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 18:19:42 +0200 Subject: [PATCH 14/15] services: treat privileged apps as granted in app_permissions Built-in apps launched without an install id (e.g. from the console) were not recognised as system apps. Use the process metadata's privilege flag as well as the install id. Co-Authored-By: Claude Fable 5.1 Signed-off-by: Joshua Jun --- src/fw/services/app_permissions/app_permissions.c | 3 ++- tests/fw/services/test_app_permissions.c | 9 +++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/fw/services/app_permissions/app_permissions.c b/src/fw/services/app_permissions/app_permissions.c index 1482378f8e..ef573d0389 100644 --- a/src/fw/services/app_permissions/app_permissions.c +++ b/src/fw/services/app_permissions/app_permissions.c @@ -54,7 +54,8 @@ AppPermissionState app_permissions_get_state_for_current_app(AppPermission permi if (!md || (permission >= AppPermissionCount)) { return AppPermissionStateNotDeclared; } - if (app_install_id_from_system(app_manager_get_current_app_id())) { + // Apps built into the firmware are trusted with everything. + if (!md->is_unprivileged || app_install_id_from_system(app_manager_get_current_app_id())) { return AppPermissionStateGranted; } return app_permissions_get_state_for_app(&md->uuid, prv_md_declares(md, permission), permission); diff --git a/tests/fw/services/test_app_permissions.c b/tests/fw/services/test_app_permissions.c index 26eee0d64c..6569c222ef 100644 --- a/tests/fw/services/test_app_permissions.c +++ b/tests/fw/services/test_app_permissions.c @@ -115,7 +115,7 @@ void test_app_permissions__initialize(void) { fake_event_init(); s_has_record = false; s_num_sets = 0; - s_md = (PebbleProcessMd){.uuid = s_uuid_app, .uses_microphone = true}; + s_md = (PebbleProcessMd){.uuid = s_uuid_app, .uses_microphone = true, .is_unprivileged = true}; s_current_md = &s_md; s_current_id = 1; s_blob_db_subscription = NULL; @@ -189,8 +189,13 @@ void test_app_permissions__current_app(void) { } void test_app_permissions__system_apps_always_granted(void) { - s_current_id = -5; // system install ids are negative s_md.uses_microphone = false; + s_current_id = -5; // system install ids are negative + cl_assert_equal_i(AppPermissionStateGranted, + app_permissions_get_state_for_current_app(AppPermission_Microphone)); + // Built-in apps launched without an install id (e.g. from the console) are privileged + s_current_id = 0; + s_md.is_unprivileged = false; cl_assert_equal_i(AppPermissionStateGranted, app_permissions_get_state_for_current_app(AppPermission_Microphone)); } From 02a48f66d972acb3dbcebd45f0ebf5d24c767037 Mon Sep 17 00:00:00 2001 From: Joshua Jun Date: Wed, 16 Sep 2026 18:40:24 +0200 Subject: [PATCH 15/15] services: stream app microphone audio to the phone mic_stream_to_phone_start() lets an app send its microphone to the phone without touching the samples: the capture service opens the Speex encoder, sets up an audio endpoint transfer and asks the phone for a new VoiceEndpointSessionTypeAudioStream session tagged with the app UUID. Once the phone accepts, frames are encoded on KernelBG and sent over the audio endpoint like dictation, with no result expected. The app hears about acceptance (started) and about the phone refusing, stopping or not answering (MicDataStopReasonPhone). Dictation preempts a stream before it takes the encoder and endpoints. SDK revision 113. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01GvaCa5cZd2nfywBVhRP9QE Signed-off-by: Joshua Jun --- docs/development/app_permissions.md | 16 ++ .../mic_capture/mic_capture_service.h | 22 +- include/pbl/services/voice_endpoint.h | 2 + src/fw/applib/mic_data_service.c | 53 +++- src/fw/applib/mic_data_service.h | 27 +- src/fw/applib/mic_data_service_private.h | 4 +- src/fw/apps/demo/mic_demo/mic_demo.c | 28 +- src/fw/kernel/events.h | 1 + .../process_management/pebble_process_info.h | 3 +- src/fw/services/mic_capture/Kconfig | 2 +- .../mic_capture/mic_capture_service.c | 241 ++++++++++++++-- src/fw/services/voice/voice.c | 7 + src/fw/services/voice_endpoint/service.c | 9 + src/fw/syscall/syscall.h | 1 + src/fw/syscall/syscall_mic_capture.c | 8 + tests/fw/services/test_mic_capture_service.c | 259 ++++++++++++++++++ .../generate_native_sdk/exported_symbols.json | 20 +- 17 files changed, 668 insertions(+), 35 deletions(-) diff --git a/docs/development/app_permissions.md b/docs/development/app_permissions.md index ea72195150..48ab642823 100644 --- a/docs/development/app_permissions.md +++ b/docs/development/app_permissions.md @@ -95,6 +95,22 @@ through the unobstructed area service, composing with Timeline Peek, so `layer_get_unobstructed_bounds()` shrinks for the app. Any modal that could hide the banner also takes focus, which is what stops capture. +## Streaming to the phone + +`mic_stream_to_phone_start()` (applib) skips the app entirely: the capture +service opens the Speex encoder, sets up an audio endpoint transfer and asks +the phone for a `VoiceEndpointSessionTypeAudioStream` session over the voice +endpoint, tagged with the app UUID (untagged for built-in apps). Once the +phone accepts, the mic starts and every frame is encoded on KernelBG and sent +over the audio endpoint (10000), exactly like dictation but with no result +expected. The app is told through `started`, and through `stopped` with +`MicDataStopReasonPhone` if the phone refuses, stops, or never answers (8 s). +Dictation preempts a stream the same way it preempts capture. + +On the phone, `VoiceSessionManager` answers the session, decodes the frames +and publishes them as PCM to the app's PebbleKit JS runtime as +`audiostream` events (see `WatchAudioStreams`). + ## Encoding Raw PCM is too much for the Bluetooth link. `audio_encoder_open()` (applib) diff --git a/include/pbl/services/mic_capture/mic_capture_service.h b/include/pbl/services/mic_capture/mic_capture_service.h index efe90997e3..46cda8d5da 100644 --- a/include/pbl/services/mic_capture/mic_capture_service.h +++ b/include/pbl/services/mic_capture/mic_capture_service.h @@ -12,8 +12,10 @@ //! //! The service owns the mic (through mic_manager), a kernel ring buffer the app drains through //! syscalls, and the OS "listening" banner. It only ever serves the app task, only while the app -//! is in focus and holds the microphone permission. Capture stops on any focus loss, when the -//! grant is revoked, when dictation preempts the mic, or when the app goes away. +//! is in focus and holds the microphone permission. Watchfaces are refused outright: they run +//! unattended for hours, so nothing they declare or the user grants lets them record. Capture stops +//! on any focus loss, when the grant is revoked, when dictation preempts the mic, or when the app +//! goes away. //! //! Data flow: the mic driver hands chunks of samples_per_update samples on KernelBG; they are //! appended to the ring buffer (dropping the newest chunk when full) and a coalesced @@ -27,6 +29,8 @@ typedef enum MicCaptureStartResult { MicCaptureStartErrNotForeground, MicCaptureStartErrInvalidArgs, MicCaptureStartErrNoMemory, + //! Watchfaces may never record + MicCaptureStartErrWatchface, } MicCaptureStartResult; typedef enum MicCaptureStopReason { @@ -36,6 +40,8 @@ typedef enum MicCaptureStopReason { MicCaptureStopReasonPermissionRevoked, MicCaptureStopReasonAppExit, MicCaptureStopReasonError, + //! The phone refused, stopped or lost the stream + MicCaptureStopReasonPhone, } MicCaptureStopReason; #define MIC_CAPTURE_SAMPLE_RATE (16000) @@ -67,3 +73,15 @@ void mic_capture_service_handle_app_focus_lost(void); //! The running app's grants changed: capture stops if the mic is no longer granted. void mic_capture_service_handle_permission_changed(void); + +//! The system (dictation) is about to take the microphone and the phone-side audio session. +void mic_capture_service_handle_system_preempt(void); + +//! Streams encoded audio straight to the phone instead of delivering PCM to the app. The +//! phone's companion for the app receives it. Same rules as capture, plus a session handshake: +//! MicCaptureEventStarted is posted once the phone accepted, and MicCaptureStopReasonPhone +//! reported if it refuses, stops, or the setup times out. +MicCaptureStartResult mic_capture_service_start_stream(PebbleTask owner); + +//! Result of the phone's answer to the stream session setup. Called by the voice endpoint. +void mic_capture_service_handle_stream_setup_result(uint8_t voice_endpoint_result); diff --git a/include/pbl/services/voice_endpoint.h b/include/pbl/services/voice_endpoint.h index 52477c9ea0..5006b31180 100644 --- a/include/pbl/services/voice_endpoint.h +++ b/include/pbl/services/voice_endpoint.h @@ -15,6 +15,8 @@ typedef enum { VoiceEndpointSessionTypeDictation = 0x01, VoiceEndpointSessionTypeCommand = 0x02, // Not used yet VoiceEndpointSessionTypeNLP = 0x03, + //! Live audio for an app; the phone decodes and hands it to the app's companion, no result + VoiceEndpointSessionTypeAudioStream = 0x04, VoiceEndpointSessionTypeCount, } VoiceEndpointSessionType; diff --git a/src/fw/applib/mic_data_service.c b/src/fw/applib/mic_data_service.c index 29fc9b8bef..1e1e822074 100644 --- a/src/fw/applib/mic_data_service.c +++ b/src/fw/applib/mic_data_service.c @@ -23,8 +23,10 @@ static void prv_teardown(MicDataServiceState *state) { applib_free(state->buffer); state->buffer = NULL; state->handlers = (MicDataHandlers){}; + state->stream_handlers = (MicStreamHandlers){}; state->context = NULL; state->active = false; + state->streaming = false; } static MicDataStopReason prv_map_stop_reason(uint8_t kernel_reason) { @@ -37,6 +39,8 @@ static MicDataStopReason prv_map_stop_reason(uint8_t kernel_reason) { return MicDataStopReasonInterrupted; case MicCaptureStopReasonPermissionRevoked: return MicDataStopReasonPermissionRevoked; + case MicCaptureStopReasonPhone: + return MicDataStopReasonPhone; case MicCaptureStopReasonAppExit: case MicCaptureStopReasonError: break; @@ -52,7 +56,8 @@ static void prv_handle_event(PebbleEvent *e, void *context) { if (e->mic_capture.type == MicCaptureEventStopped) { const MicDataStopReason reason = prv_map_stop_reason(e->mic_capture.stop_reason); - const MicDataStoppedHandler stopped = state->handlers.stopped; + const MicDataStoppedHandler stopped = + state->streaming ? state->stream_handlers.stopped : state->handlers.stopped; void *ctx = state->context; prv_teardown(state); if (stopped) { @@ -60,6 +65,15 @@ static void prv_handle_event(PebbleEvent *e, void *context) { } return; } + if (e->mic_capture.type == MicCaptureEventStarted) { + if (state->streaming && state->stream_handlers.started) { + state->stream_handlers.started(state->context); + } + return; + } + if (state->streaming) { + return; + } bool overrun = e->mic_capture.overrun; while (state->active && (sys_mic_capture_get_available() >= state->samples_per_update)) { @@ -120,6 +134,43 @@ bool mic_data_service_is_active(void) { return state ? state->active : false; } +MicDataStartResult mic_stream_to_phone_start(MicStreamHandlers handlers, void *context) { + MicDataServiceState *state = prv_get_state(); + if (!state) { + return MicDataStartErrNotForeground; + } + if (!handlers.stopped) { + return MicDataStartErrInvalidArgs; + } + if (state->active) { + return MicDataStartErrBusy; + } + const MicDataStartResult rv = (MicDataStartResult)sys_mic_capture_start_stream(); + if (rv != MicDataStartOk) { + return rv; + } + state->stream_handlers = handlers; + state->context = context; + state->active = true; + state->streaming = true; + event_service_client_subscribe(&state->event_info); + return MicDataStartOk; +} + +void mic_stream_to_phone_stop(void) { + MicDataServiceState *state = prv_get_state(); + if (!state || !state->active || !state->streaming) { + return; + } + sys_mic_capture_stop(); + prv_teardown(state); +} + +bool mic_stream_to_phone_is_active(void) { + MicDataServiceState *state = prv_get_state(); + return state ? (state->active && state->streaming) : false; +} + void mic_data_service_state_init(MicDataServiceState *state) { *state = (MicDataServiceState){ .event_info = { diff --git a/src/fw/applib/mic_data_service.h b/src/fw/applib/mic_data_service.h index 48bf2aae4b..5a06cdb170 100644 --- a/src/fw/applib/mic_data_service.h +++ b/src/fw/applib/mic_data_service.h @@ -59,6 +59,8 @@ typedef enum MicDataStopReason { MicDataStopReasonPermissionRevoked, //! An unexpected error MicDataStopReasonError, + //! The phone refused, stopped or lost the stream (streaming only) + MicDataStopReasonPhone, } MicDataStopReason; //! Handler receiving a batch of samples. @@ -94,8 +96,31 @@ MicDataStartResult mic_data_service_subscribe(uint32_t samples_per_update, MicDa //! Stops capturing. Safe to call when not capturing. void mic_data_service_unsubscribe(void); -//! @return true while the app is capturing +//! @return true while the app is capturing or streaming bool mic_data_service_is_active(void); +//! Handler called once the phone has accepted the stream and audio is flowing. +typedef void (*MicStreamStartedHandler)(void *context); + +typedef struct MicStreamHandlers { + MicStreamStartedHandler started; + MicDataStoppedHandler stopped; +} MicStreamHandlers; + +//! Streams the microphone straight to the phone, encoded (Speex wideband, ~10 kbps), instead of +//! delivering samples to the app. The phone decodes it and hands the PCM to the app's +//! companion (PebbleKit JS `audiostream` events). The same rules as +//! \ref mic_data_service_subscribe apply; in addition the phone has to accept the stream, which +//! is reported through `started`, and can end it, reported as \ref MicDataStopReasonPhone. +//! @param handlers `stopped` is required +//! @param context Passed to the handlers +MicDataStartResult mic_stream_to_phone_start(MicStreamHandlers handlers, void *context); + +//! Stops streaming. Safe to call when not streaming. +void mic_stream_to_phone_stop(void); + +//! @return true while the app is streaming to the phone (including the setup handshake) +bool mic_stream_to_phone_is_active(void); + //! @} // end addtogroup Microphone //! @} // end addtogroup Foundation diff --git a/src/fw/applib/mic_data_service_private.h b/src/fw/applib/mic_data_service_private.h index 28375e361a..da542b8017 100644 --- a/src/fw/applib/mic_data_service_private.h +++ b/src/fw/applib/mic_data_service_private.h @@ -9,10 +9,12 @@ typedef struct MicDataServiceState { EventServiceInfo event_info; MicDataHandlers handlers; + MicStreamHandlers stream_handlers; void *context; - int16_t *buffer; //!< samples_per_update samples, owned by the subscription + int16_t *buffer; //!< samples_per_update samples, owned by a capture subscription uint32_t samples_per_update; bool active; + bool streaming; //!< active as a stream to the phone rather than a capture } MicDataServiceState; void mic_data_service_state_init(MicDataServiceState *state); diff --git a/src/fw/apps/demo/mic_demo/mic_demo.c b/src/fw/apps/demo/mic_demo/mic_demo.c index 3f86e6aed8..846e594430 100644 --- a/src/fw/apps/demo/mic_demo/mic_demo.c +++ b/src/fw/apps/demo/mic_demo/mic_demo.c @@ -14,8 +14,9 @@ #include #include -// Exercises the Microphone API: SELECT toggles capture, the screen shows the permission state, -// the RMS level of the last batch and the unobstructed height (which shrinks under the banner). +// Exercises the Microphone API: SELECT toggles capture, UP toggles streaming to the phone. The +// screen shows the permission state, the RMS level of the last batch and the unobstructed height +// (which shrinks under the banner). #define SAMPLES_PER_UPDATE (320) // 20 ms @@ -75,6 +76,7 @@ static void prv_stopped_handler(MicDataStopReason reason, void *context) { [MicDataStopReasonInterrupted] = "interrupted", [MicDataStopReasonPermissionRevoked] = "revoked", [MicDataStopReasonError] = "error", + [MicDataStopReasonPhone] = "phone ended", }; PBL_LOG_DBG("mic demo: capture stopped (%s)", s_reasons[reason]); prv_update_status(data, s_reasons[reason]); @@ -110,8 +112,27 @@ static void prv_select_click_handler(ClickRecognizerRef recognizer, void *contex prv_update_status(data, s_results[rv]); } +static void prv_stream_started(void *context) { + PBL_LOG_DBG("mic demo: phone accepted the stream"); + prv_update_status(context, "streaming"); +} + +static void prv_up_click_handler(ClickRecognizerRef recognizer, void *context) { + MicDemoAppData *data = context; + if (mic_stream_to_phone_is_active()) { + mic_stream_to_phone_stop(); + prv_update_status(data, "idle"); + return; + } + const MicDataStartResult rv = mic_stream_to_phone_start( + (MicStreamHandlers){.started = prv_stream_started, .stopped = prv_stopped_handler}, data); + PBL_LOG_DBG("mic demo: stream -> %u", rv); + prv_update_status(data, (rv == MicDataStartOk) ? "waiting for phone" : "stream refused"); +} + static void prv_click_config_provider(void *context) { window_single_click_subscribe(BUTTON_ID_SELECT, prv_select_click_handler); + window_single_click_subscribe(BUTTON_ID_UP, prv_up_click_handler); } static void prv_unobstructed_did_change(void *context) { @@ -131,7 +152,7 @@ static void prv_window_load(Window *window) { text_layer_set_text_alignment(&data->level_layer, GTextAlignmentCenter); layer_add_child(&window->layer, &data->level_layer.layer); - prv_update_status(data, "press SELECT"); + prv_update_status(data, "SELECT: capture UP: stream"); } static void prv_handle_init(void) { @@ -153,6 +174,7 @@ static void prv_handle_init(void) { static void prv_handle_deinit(void) { MicDemoAppData *data = app_state_get_user_data(); mic_data_service_unsubscribe(); + mic_stream_to_phone_stop(); app_permission_service_unsubscribe(); app_unobstructed_area_service_unsubscribe(); app_free(data); diff --git a/src/fw/kernel/events.h b/src/fw/kernel/events.h index f0fb5b34a5..a552fc0a81 100644 --- a/src/fw/kernel/events.h +++ b/src/fw/kernel/events.h @@ -508,6 +508,7 @@ typedef struct PACKED { // 2 bytes typedef enum { MicCaptureEventData = 0, MicCaptureEventStopped = 1, + MicCaptureEventStarted = 2, //!< A stream to the phone is up } MicCaptureEventType; typedef struct PACKED { // 5 bytes diff --git a/src/fw/process_management/pebble_process_info.h b/src/fw/process_management/pebble_process_info.h index cb986de201..8a63ba4629 100644 --- a/src/fw/process_management/pebble_process_info.h +++ b/src/fw/process_management/pebble_process_info.h @@ -173,9 +173,10 @@ typedef enum { // sdk.major:0x5 .minor:0x6b -- Add app permissions API (rev 110) // sdk.major:0x5 .minor:0x6c -- Add Microphone API (mic_data_service_subscribe) (rev 111) // sdk.major:0x5 .minor:0x6d -- Add AudioEncoder API (audio_encoder_open) (rev 112) +// sdk.major:0x5 .minor:0x6e -- Add mic_stream_to_phone_start() (rev 113) #define PROCESS_INFO_CURRENT_SDK_VERSION_MAJOR 0x5 -#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6d +#define PROCESS_INFO_CURRENT_SDK_VERSION_MINOR 0x6e // The first SDK to ship with 2.x APIs #define PROCESS_INFO_FIRST_2X_SDK_VERSION_MAJOR 0x4 diff --git a/src/fw/services/mic_capture/Kconfig b/src/fw/services/mic_capture/Kconfig index 9b36d81b93..5bbc0c17a0 100644 --- a/src/fw/services/mic_capture/Kconfig +++ b/src/fw/services/mic_capture/Kconfig @@ -3,7 +3,7 @@ config SERVICE_MIC_CAPTURE bool "Microphone capture for apps" - depends on SERVICE_MIC_MANAGER && SERVICE_APP_PERMISSIONS + depends on SERVICE_MIC_MANAGER && SERVICE_APP_PERMISSIONS && SERVICE_AUDIO_ENCODER default y help Live PCM capture service backing the app microphone API. diff --git a/src/fw/services/mic_capture/mic_capture_service.c b/src/fw/services/mic_capture/mic_capture_service.c index 1d796fad4e..c08fda5dcb 100644 --- a/src/fw/services/mic_capture/mic_capture_service.c +++ b/src/fw/services/mic_capture/mic_capture_service.c @@ -9,30 +9,53 @@ #include "kernel/ui/modals/modal_manager.h" #include "pbl/kernel/mutex.h" #include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/audio_encoder/audio_encoder.h" +#include "pbl/services/audio_endpoint.h" #include "pbl/services/mic_manager.h" +#include "pbl/services/new_timer/new_timer.h" +#include "pbl/services/voice_endpoint.h" #include "pbl/util/circular_buffer.h" #include "popups/mic_banner.h" +#include "process_management/app_install_manager.h" +#include "process_management/app_manager.h" #include #include PBL_LOG_MODULE_DEFINE(service_mic_capture, CONFIG_SERVICE_MIC_CAPTURE_LOG_LEVEL); -#define RING_BYTES (MIC_CAPTURE_RING_SAMPLES * sizeof(int16_t)) +#define RING_BYTES (MIC_CAPTURE_RING_SAMPLES * sizeof(int16_t)) +#define STREAM_SETUP_TIMEOUT_MS (8000) + +typedef enum { + MicCaptureSinkApp = 0, //!< PCM batches to the app + MicCaptureSinkPhone, //!< Encoded frames to the phone +} MicCaptureSink; typedef struct { bool active; PebbleTask owner; + MicCaptureSink sink; uint16_t samples_per_update; + int16_t *chunk; //!< The mic driver fills this, samples_per_update samples + + // App sink CircularBuffer ring; uint8_t *ring_storage; - int16_t *chunk; bool overrun; bool data_event_pending; + + // Phone sink + bool stream_setup_pending; //!< Waiting for the phone to accept the session + bool stream_transfer_open; //!< audio_endpoint transfer set up + AudioEndpointSessionId stream_session; + AudioEncoderInfo encoder; + uint8_t *packet; //!< encoder.max_packet_bytes } MicCaptureState; static PBL_MUTEX_DEFINE(s_lock); static MicCaptureState s_state; +static TimerID s_setup_timer = TIMER_INVALID_ID; // The banner is UI, so it is driven from KernelMain whichever task stops or starts capture. static void prv_show_banner_cb(void *unused) { @@ -45,6 +68,7 @@ static void prv_hide_banner_cb(void *unused) { void mic_capture_service_init(void) { s_state = (MicCaptureState){}; + s_setup_timer = new_timer_create(); } static void prv_post_event(uint8_t type, MicCaptureStopReason reason, bool overrun, @@ -61,15 +85,23 @@ static void prv_post_event(uint8_t type, MicCaptureStopReason reason, bool overr event_put(&e); } -//! Expects s_lock held. Frees resources and returns the buffers to free. -static void prv_teardown_locked(void) { - s_state.active = false; - s_state.owner = PebbleTask_Unknown; - s_state.data_event_pending = false; +//! Expects s_lock held. Frees resources and closes the phone session, if any. +static void prv_teardown_locked(bool phone_still_expects_stop) { + if (s_state.sink == MicCaptureSinkPhone) { + new_timer_stop(s_setup_timer); + if (s_state.stream_transfer_open) { + if (phone_still_expects_stop) { + audio_endpoint_stop_transfer(s_state.stream_session); + } else { + audio_endpoint_cancel_transfer(s_state.stream_session); + } + } + audio_encoder_service_close(s_state.owner); + } kernel_free(s_state.ring_storage); kernel_free(s_state.chunk); - s_state.ring_storage = NULL; - s_state.chunk = NULL; + kernel_free(s_state.packet); + s_state = (MicCaptureState){}; launcher_task_add_callback(prv_hide_banner_cb, NULL); } @@ -82,7 +114,7 @@ static void prv_stop_with_reason(MicCaptureStopReason reason, bool release_mic) return; } PBL_LOG_DBG("Capture stopped, reason %u", reason); - prv_teardown_locked(); + prv_teardown_locked(reason != MicCaptureStopReasonPhone); pbl_mutex_unlock(&s_lock); if (release_mic) { @@ -98,6 +130,17 @@ static void prv_mic_data_handler(int16_t *samples, size_t sample_count, void *co pbl_mutex_unlock(&s_lock); return; } + + if (s_state.sink == MicCaptureSinkPhone) { + const int len = audio_encoder_service_encode(s_state.owner, samples, sample_count, + s_state.packet, s_state.encoder.max_packet_bytes); + if (len > 0) { + audio_endpoint_add_frame(s_state.stream_session, s_state.packet, (uint8_t)len); + } + pbl_mutex_unlock(&s_lock); + return; + } + const uint16_t bytes = sample_count * sizeof(int16_t); if (!circular_buffer_write(&s_state.ring, samples, bytes)) { // Full: drop the newest chunk, the app is not keeping up @@ -128,13 +171,13 @@ static bool prv_app_is_in_focus(void) { return !modal_manager_get_enabled() || (modal_manager_get_properties() & ModalProperty_Unfocused); } -MicCaptureStartResult mic_capture_service_start(PebbleTask owner, uint16_t samples_per_update) { +//! Common admission checks. Expects s_lock NOT held. +static MicCaptureStartResult prv_check_start(PebbleTask owner) { if (owner != PebbleTask_App) { return MicCaptureStartErrNotForeground; } - if ((samples_per_update < MIC_CAPTURE_MIN_SAMPLES_PER_UPDATE) || - (samples_per_update > MIC_CAPTURE_MAX_SAMPLES_PER_UPDATE)) { - return MicCaptureStartErrInvalidArgs; + if (app_manager_is_watchface_running()) { + return MicCaptureStartErrWatchface; } if (!prv_app_is_in_focus()) { return MicCaptureStartErrNotForeground; @@ -147,6 +190,28 @@ MicCaptureStartResult mic_capture_service_start(PebbleTask owner, uint16_t sampl case AppPermissionStateGranted: break; } + return MicCaptureStartOk; +} + +//! Expects s_lock held and s_state prepared. Starts the mic and shows the banner. +static bool prv_acquire_mic_locked(void) { + if (!mic_manager_acquire(MicClientAppCapture, prv_mic_data_handler, NULL, s_state.chunk, + s_state.samples_per_update, prv_preempted, NULL)) { + return false; + } + launcher_task_add_callback(prv_show_banner_cb, NULL); + return true; +} + +MicCaptureStartResult mic_capture_service_start(PebbleTask owner, uint16_t samples_per_update) { + if ((samples_per_update < MIC_CAPTURE_MIN_SAMPLES_PER_UPDATE) || + (samples_per_update > MIC_CAPTURE_MAX_SAMPLES_PER_UPDATE)) { + return MicCaptureStartErrInvalidArgs; + } + const MicCaptureStartResult check = prv_check_start(owner); + if (check != MicCaptureStartOk) { + return check; + } pbl_mutex_lock(&s_lock, PBL_FOREVER); if (s_state.active) { @@ -166,33 +231,156 @@ MicCaptureStartResult mic_capture_service_start(PebbleTask owner, uint16_t sampl s_state = (MicCaptureState){ .active = true, .owner = owner, + .sink = MicCaptureSinkApp, .samples_per_update = samples_per_update, .ring_storage = ring_storage, .chunk = chunk, }; circular_buffer_init(&s_state.ring, ring_storage, RING_BYTES); - pbl_mutex_unlock(&s_lock); - if (!mic_manager_acquire(MicClientAppCapture, prv_mic_data_handler, NULL, chunk, - samples_per_update, prv_preempted, NULL)) { - pbl_mutex_lock(&s_lock, PBL_FOREVER); - prv_teardown_locked(); + if (!prv_acquire_mic_locked()) { + prv_teardown_locked(false); pbl_mutex_unlock(&s_lock); return MicCaptureStartErrBusy; } + pbl_mutex_unlock(&s_lock); - launcher_task_add_callback(prv_show_banner_cb, NULL); PBL_LOG_DBG("Capture started, %u samples per update", samples_per_update); return MicCaptureStartOk; } +// Stream to phone +//////////////////////////////////////////////////////////////////////////////// + +static void prv_stream_setup_timeout(void *data) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + const bool pending = s_state.active && s_state.stream_setup_pending; + pbl_mutex_unlock(&s_lock); + if (pending) { + PBL_LOG_WRN("Phone did not answer the audio stream setup"); + prv_stop_with_reason(MicCaptureStopReasonPhone, false /* mic never started */); + } +} + +static void prv_stream_transfer_stopped(AudioEndpointSessionId session_id) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + const bool ours = s_state.active && (s_state.sink == MicCaptureSinkPhone) && + (s_state.stream_session == session_id); + if (ours) { + // The endpoint already closed the session on its side + s_state.stream_transfer_open = false; + } + const bool mic_running = ours && !s_state.stream_setup_pending; + pbl_mutex_unlock(&s_lock); + if (ours) { + PBL_LOG_DBG("Phone stopped the audio stream"); + prv_stop_with_reason(MicCaptureStopReasonPhone, mic_running); + } +} + +MicCaptureStartResult mic_capture_service_start_stream(PebbleTask owner) { + const MicCaptureStartResult check = prv_check_start(owner); + if (check != MicCaptureStartOk) { + return check; + } + if (!audio_encoder_service_is_codec_available(AudioCodecSpeexWB)) { + return MicCaptureStartErrBusy; + } + + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (s_state.active || (mic_manager_get_owner() != MicClientNone)) { + pbl_mutex_unlock(&s_lock); + return MicCaptureStartErrBusy; + } + + AudioEncoderInfo info; + if (!audio_encoder_service_open(AudioCodecSpeexWB, owner, &info)) { + pbl_mutex_unlock(&s_lock); + return MicCaptureStartErrBusy; + } + const uint16_t samples_per_update = info.frame_samples * info.channels; + int16_t *chunk = kernel_malloc(samples_per_update * sizeof(int16_t)); + uint8_t *packet = kernel_malloc(info.max_packet_bytes); + if (!chunk || !packet) { + kernel_free(chunk); + kernel_free(packet); + audio_encoder_service_close(owner); + pbl_mutex_unlock(&s_lock); + return MicCaptureStartErrNoMemory; + } + s_state = (MicCaptureState){ + .active = true, + .owner = owner, + .sink = MicCaptureSinkPhone, + .samples_per_update = samples_per_update, + .chunk = chunk, + .packet = packet, + .encoder = info, + .stream_setup_pending = true, + }; + s_state.stream_session = audio_endpoint_setup_transfer(prv_stream_transfer_stopped); + s_state.stream_transfer_open = true; + + AudioTransferInfoSpeex transfer_info = { + .sample_rate = info.sample_rate, + .bit_rate = (uint16_t)info.bitrate, + .frame_size = info.frame_samples, + .bitstream_version = info.bitstream_version, + }; + strncpy(transfer_info.version, "1.2.1", sizeof(transfer_info.version) - 1); + + const PebbleProcessMd *md = app_manager_get_current_app_md(); + Uuid app_uuid = md ? md->uuid : UUID_INVALID; + const bool from_app = + md && !app_install_id_from_system(app_manager_get_current_app_id()) && md->is_unprivileged; + voice_endpoint_setup_session(VoiceEndpointSessionTypeAudioStream, s_state.stream_session, + &transfer_info, from_app ? &app_uuid : NULL); + new_timer_start(s_setup_timer, STREAM_SETUP_TIMEOUT_MS, prv_stream_setup_timeout, NULL, 0); + pbl_mutex_unlock(&s_lock); + + PBL_LOG_DBG("Audio stream requested, session %u", s_state.stream_session); + return MicCaptureStartOk; +} + +void mic_capture_service_handle_stream_setup_result(uint8_t voice_endpoint_result) { + pbl_mutex_lock(&s_lock, PBL_FOREVER); + if (!s_state.active || (s_state.sink != MicCaptureSinkPhone) || !s_state.stream_setup_pending) { + pbl_mutex_unlock(&s_lock); + return; + } + new_timer_stop(s_setup_timer); + s_state.stream_setup_pending = false; + + if (voice_endpoint_result != VoiceEndpointResultSuccess) { + PBL_LOG_WRN("Phone refused the audio stream (%u)", voice_endpoint_result); + prv_teardown_locked(false); + pbl_mutex_unlock(&s_lock); + prv_post_event(MicCaptureEventStopped, MicCaptureStopReasonPhone, false, 0); + return; + } + + if (!prv_acquire_mic_locked()) { + prv_teardown_locked(true); + pbl_mutex_unlock(&s_lock); + prv_post_event(MicCaptureEventStopped, MicCaptureStopReasonError, false, 0); + return; + } + pbl_mutex_unlock(&s_lock); + + PBL_LOG_DBG("Audio stream started"); + prv_post_event(MicCaptureEventStarted, MicCaptureStopReasonStopped, false, 0); +} + +// Stopping +//////////////////////////////////////////////////////////////////////////////// + static void prv_stop_silently(PebbleTask task) { pbl_mutex_lock(&s_lock, PBL_FOREVER); if (!s_state.active || (s_state.owner != task)) { pbl_mutex_unlock(&s_lock); return; } - prv_teardown_locked(); + prv_teardown_locked(true); pbl_mutex_unlock(&s_lock); mic_manager_release(MicClientAppCapture); PBL_LOG_DBG("Capture stopped"); @@ -208,7 +396,7 @@ void mic_capture_service_stop_for_task(PebbleTask task) { uint32_t mic_capture_service_read(PebbleTask owner, int16_t *out, uint32_t max_samples) { pbl_mutex_lock(&s_lock, PBL_FOREVER); - if (!s_state.active || (s_state.owner != owner) || !out) { + if (!s_state.active || (s_state.owner != owner) || (s_state.sink != MicCaptureSinkApp) || !out) { pbl_mutex_unlock(&s_lock); return 0; } @@ -228,8 +416,9 @@ uint32_t mic_capture_service_read(PebbleTask owner, int16_t *out, uint32_t max_s uint32_t mic_capture_service_get_available(void) { pbl_mutex_lock(&s_lock, PBL_FOREVER); const uint32_t available = - s_state.active ? circular_buffer_get_read_space_remaining(&s_state.ring) / sizeof(int16_t) - : 0; + (s_state.active && (s_state.sink == MicCaptureSinkApp)) + ? circular_buffer_get_read_space_remaining(&s_state.ring) / sizeof(int16_t) + : 0; pbl_mutex_unlock(&s_lock); return available; } @@ -250,3 +439,7 @@ void mic_capture_service_handle_permission_changed(void) { prv_stop_with_reason(MicCaptureStopReasonPermissionRevoked, true); } } + +void mic_capture_service_handle_system_preempt(void) { + prv_stop_with_reason(MicCaptureStopReasonPreempted, true); +} diff --git a/src/fw/services/voice/voice.c b/src/fw/services/voice/voice.c index 44887ffcd6..7459238a57 100644 --- a/src/fw/services/voice/voice.c +++ b/src/fw/services/voice/voice.c @@ -12,6 +12,9 @@ #include "process_management/app_manager.h" #include "pbl/services/comm_session/session.h" #include "pbl/services/mic_manager.h" +#ifdef CONFIG_SERVICE_MIC_CAPTURE +#include "pbl/services/mic_capture/mic_capture_service.h" +#endif #include "pbl/services/new_timer/new_timer.h" #include "pbl/services/audio_endpoint.h" #include "pbl/services/voice/transcription.h" @@ -339,6 +342,10 @@ void voice_init(void) { // prv_session_setup_timeout) VoiceSessionId voice_start_dictation(VoiceEndpointSessionType session_type) { PBL_LOG_DBG("voice_start_dictation called with session_type: %d", session_type); +#ifdef CONFIG_SERVICE_MIC_CAPTURE + // Dictation takes the mic, the encoder and the phone-side audio session away from any app. + mic_capture_service_handle_system_preempt(); +#endif pbl_mutex_lock(&s_lock, PBL_FOREVER); // Lazily initialize Speex encoder to avoid baseline memory usage when voice not used diff --git a/src/fw/services/voice_endpoint/service.c b/src/fw/services/voice_endpoint/service.c index 4026f295b5..2c24aceb05 100644 --- a/src/fw/services/voice_endpoint/service.c +++ b/src/fw/services/voice_endpoint/service.c @@ -2,6 +2,9 @@ /* SPDX-License-Identifier: Apache-2.0 */ #include "pbl/services/voice_endpoint.h" +#ifdef CONFIG_SERVICE_MIC_CAPTURE +#include "pbl/services/mic_capture/mic_capture_service.h" +#endif #include "kernel/pbl_malloc.h" #include "pbl/services/comm_session/session.h" @@ -142,6 +145,12 @@ void voice_endpoint_protocol_msg_callback(CommSession *session, const uint8_t *d } bool app_initiated = (msg->flags.app_initiated == 1); +#ifdef CONFIG_SERVICE_MIC_CAPTURE + if (msg->session_type == VoiceEndpointSessionTypeAudioStream) { + mic_capture_service_handle_stream_setup_result(result); + break; + } +#endif voice_handle_session_setup_result(result, msg->session_type, app_initiated); } else { PBL_LOG_WRN("Invalid size for session setup result message"); diff --git a/src/fw/syscall/syscall.h b/src/fw/syscall/syscall.h index 327c62b082..7c6c37a680 100644 --- a/src/fw/syscall/syscall.h +++ b/src/fw/syscall/syscall.h @@ -109,6 +109,7 @@ uint8_t sys_app_permission_get_state(uint8_t permission); //! Live microphone capture for the app task. See mic_capture_service.h. uint8_t sys_mic_capture_start(uint16_t samples_per_update); +uint8_t sys_mic_capture_start_stream(void); void sys_mic_capture_stop(void); uint32_t sys_mic_capture_read(int16_t *out, uint32_t max_samples); uint32_t sys_mic_capture_get_available(void); diff --git a/src/fw/syscall/syscall_mic_capture.c b/src/fw/syscall/syscall_mic_capture.c index 2f3469fbba..40a125a47a 100644 --- a/src/fw/syscall/syscall_mic_capture.c +++ b/src/fw/syscall/syscall_mic_capture.c @@ -17,6 +17,14 @@ DEFINE_SYSCALL(uint8_t, sys_mic_capture_start, uint16_t samples_per_update) { #endif } +DEFINE_SYSCALL(uint8_t, sys_mic_capture_start_stream, void) { +#ifdef CONFIG_SERVICE_MIC_CAPTURE + return (uint8_t)mic_capture_service_start_stream(pebble_task_get_current()); +#else + return (uint8_t)MicCaptureStartErrDenied; +#endif +} + DEFINE_SYSCALL(void, sys_mic_capture_stop, void) { #ifdef CONFIG_SERVICE_MIC_CAPTURE mic_capture_service_stop(pebble_task_get_current()); diff --git a/tests/fw/services/test_mic_capture_service.c b/tests/fw/services/test_mic_capture_service.c index 96b306d8f9..d57a0d9cb1 100644 --- a/tests/fw/services/test_mic_capture_service.c +++ b/tests/fw/services/test_mic_capture_service.c @@ -6,8 +6,13 @@ #include "kernel/events.h" #include "kernel/ui/modals/modal_manager.h" #include "pbl/services/app_permissions/app_permissions.h" +#include "pbl/services/audio_encoder/audio_encoder.h" +#include "pbl/services/audio_endpoint.h" #include "pbl/services/mic_capture/mic_capture_service.h" #include "pbl/services/mic_manager.h" +#include "pbl/services/voice_endpoint.h" +#include "process_management/app_manager.h" +#include "process_management/pebble_process_md.h" #include @@ -23,6 +28,123 @@ #include "stubs_passert.h" #include "stubs_event_loop.h" #include "stubs_mic_banner.h" +#include "fake_new_timer.h" + +// Current app +//////////////////////////////////////////////////////////////// + +static PebbleProcessMd s_md = {.is_unprivileged = true}; + +const PebbleProcessMd *app_manager_get_current_app_md(void) { + return &s_md; +} + +AppInstallId app_manager_get_current_app_id(void) { + return 1; +} + +bool app_install_id_from_system(AppInstallId id) { + return (id < 0); +} + +static bool s_watchface_running; + +bool app_manager_is_watchface_running(void) { + return s_watchface_running; +} + +// Fake encoder + endpoints for the phone sink +//////////////////////////////////////////////////////////////// + +#define FAKE_FRAME_SAMPLES (320) + +static bool s_encoder_open; +static int s_encoder_open_count; +static int s_encoder_close_count; +static int s_encode_count; + +bool audio_encoder_service_is_codec_available(AudioCodec codec) { + return (codec == AudioCodecSpeexWB); +} + +bool audio_encoder_service_open(AudioCodec codec, PebbleTask owner, AudioEncoderInfo *info_out) { + if (s_encoder_open) { + return false; + } + s_encoder_open = true; + s_encoder_open_count++; + *info_out = (AudioEncoderInfo){ + .codec = codec, + .channels = 1, + .frame_samples = FAKE_FRAME_SAMPLES, + .sample_rate = 16000, + .bitrate = 9800, + .max_packet_bytes = 200, + .bitstream_version = 4, + }; + return true; +} + +int audio_encoder_service_encode(PebbleTask owner, const int16_t *pcm, uint32_t num_samples, + uint8_t *out, uint32_t out_len) { + cl_assert(s_encoder_open); + cl_assert_equal_i(num_samples, FAKE_FRAME_SAMPLES); + s_encode_count++; + out[0] = (uint8_t)pcm[0]; + return 25; +} + +void audio_encoder_service_close(PebbleTask owner) { + if (s_encoder_open) { + s_encoder_close_count++; + } + s_encoder_open = false; +} + +static AudioEndpointStopTransferCallback s_transfer_stop_cb; +static AudioEndpointSessionId s_transfer_session; +static int s_frames_sent; +static uint8_t s_last_frame_byte; +static int s_transfer_stopped_count; +static int s_transfer_cancelled_count; + +AudioEndpointSessionId audio_endpoint_setup_transfer(AudioEndpointStopTransferCallback stop_cb) { + s_transfer_stop_cb = stop_cb; + return ++s_transfer_session; +} + +void audio_endpoint_add_frame(AudioEndpointSessionId session_id, uint8_t *frame, + uint8_t frame_size) { + cl_assert_equal_i(session_id, s_transfer_session); + cl_assert_equal_i(frame_size, 25); + s_frames_sent++; + s_last_frame_byte = frame[0]; +} + +void audio_endpoint_stop_transfer(AudioEndpointSessionId session_id) { + cl_assert_equal_i(session_id, s_transfer_session); + s_transfer_stopped_count++; +} + +void audio_endpoint_cancel_transfer(AudioEndpointSessionId session_id) { + cl_assert_equal_i(session_id, s_transfer_session); + s_transfer_cancelled_count++; +} + +static int s_setup_sessions; +static VoiceEndpointSessionType s_setup_type; +static bool s_setup_had_uuid; +static uint16_t s_setup_frame_size; + +void voice_endpoint_setup_session(VoiceEndpointSessionType session_type, + AudioEndpointSessionId session_id, AudioTransferInfoSpeex *info, + Uuid *app_uuid) { + s_setup_sessions++; + s_setup_type = session_type; + s_setup_had_uuid = (app_uuid != NULL); + s_setup_frame_size = info->frame_size; + cl_assert_equal_i(session_id, s_transfer_session); +} // Fake mic driver, driven by the tests //////////////////////////////////////////////////////////////// @@ -126,6 +248,19 @@ void test_mic_capture_service__initialize(void) { s_mic_handler = NULL; s_permission_state = AppPermissionStateGranted; s_modal_focused = false; + s_md = (PebbleProcessMd){.is_unprivileged = true}; + s_watchface_running = false; + s_encoder_open = false; + s_encoder_open_count = 0; + s_encoder_close_count = 0; + s_encode_count = 0; + s_transfer_stop_cb = NULL; + s_frames_sent = 0; + s_transfer_stopped_count = 0; + s_transfer_cancelled_count = 0; + s_setup_sessions = 0; + s_setup_had_uuid = false; + stub_new_timer_cleanup(); mic_manager_init(); mic_capture_service_init(); } @@ -134,6 +269,7 @@ void test_mic_capture_service__cleanup(void) { mic_capture_service_stop_for_task(PebbleTask_App); mic_manager_release(MicClientVoiceDictation); fake_mutex_assert_all_unlocked(); + stub_new_timer_cleanup(); fake_pbl_malloc_check_net_allocs(); } @@ -155,6 +291,12 @@ void test_mic_capture_service__start_refusals(void) { mic_capture_service_start(PebbleTask_App, SPU)); s_modal_focused = false; + // Watchfaces are refused even with the permission granted, for capture and streaming alike + s_watchface_running = true; + cl_assert_equal_i(MicCaptureStartErrWatchface, mic_capture_service_start(PebbleTask_App, SPU)); + cl_assert_equal_i(MicCaptureStartErrWatchface, mic_capture_service_start_stream(PebbleTask_App)); + s_watchface_running = false; + s_permission_state = AppPermissionStateNotDeclared; cl_assert_equal_i(MicCaptureStartErrNotDeclared, mic_capture_service_start(PebbleTask_App, SPU)); s_permission_state = AppPermissionStateDenied; @@ -317,3 +459,120 @@ void test_mic_capture_service__stop_for_task(void) { // Can start again afterwards cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); } + +// Streaming to the phone +//////////////////////////////////////////////////////////////// + +static void prv_start_stream_ok(void) { + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start_stream(PebbleTask_App)); + cl_assert(mic_capture_service_is_active()); + // Session requested, mic not started until the phone answers + cl_assert_equal_i(1, s_setup_sessions); + cl_assert_equal_i(s_setup_type, VoiceEndpointSessionTypeAudioStream); + cl_assert_equal_i(s_setup_frame_size, FAKE_FRAME_SAMPLES); + cl_assert_equal_i(1, s_encoder_open_count); + cl_assert(!s_mic_running); + cl_assert_equal_i(0, fake_event_get_count()); +} + +void test_mic_capture_service__stream_setup_and_data(void) { + prv_start_stream_ok(); + cl_assert(s_setup_had_uuid); // a third-party app tags the session with its UUID + + mic_capture_service_handle_stream_setup_result(VoiceEndpointResultSuccess); + cl_assert(s_mic_running); + cl_assert_equal_i(s_mic_buffer_len, FAKE_FRAME_SAMPLES); + cl_assert_equal_i(1, fake_event_get_count()); + PebbleEvent e = prv_last_event(); + cl_assert_equal_i(e.mic_capture.type, MicCaptureEventStarted); + + // Chunks are encoded and forwarded, never buffered for the app + prv_deliver_chunk(42); + prv_deliver_chunk(43); + cl_assert_equal_i(2, s_encode_count); + cl_assert_equal_i(2, s_frames_sent); + cl_assert_equal_i(43, s_last_frame_byte); + cl_assert_equal_i(0, mic_capture_service_get_available()); + int16_t out[8]; + cl_assert_equal_i(0, mic_capture_service_read(PebbleTask_App, out, 8)); + cl_assert_equal_i(1, fake_event_get_count()); + + // A duplicate setup answer is ignored + mic_capture_service_handle_stream_setup_result(VoiceEndpointResultSuccess); + cl_assert_equal_i(1, fake_event_get_count()); + + // App stops: transfer is ended politely, encoder closed, no event + mic_capture_service_stop(PebbleTask_App); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + cl_assert_equal_i(1, s_transfer_stopped_count); + cl_assert_equal_i(0, s_transfer_cancelled_count); + cl_assert_equal_i(1, s_encoder_close_count); + cl_assert_equal_i(1, fake_event_get_count()); +} + +void test_mic_capture_service__stream_refused_by_phone(void) { + prv_start_stream_ok(); + mic_capture_service_handle_stream_setup_result(VoiceEndpointResultFailDisabled); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + cl_assert_equal_i(1, s_transfer_cancelled_count); + cl_assert_equal_i(1, s_encoder_close_count); + prv_assert_stopped_event(MicCaptureStopReasonPhone); +} + +void test_mic_capture_service__stream_setup_timeout(void) { + prv_start_stream_ok(); + cl_assert(stub_new_timer_fire(stub_new_timer_get_next())); + cl_assert(!mic_capture_service_is_active()); + cl_assert_equal_i(1, s_transfer_cancelled_count); + prv_assert_stopped_event(MicCaptureStopReasonPhone); +} + +void test_mic_capture_service__stream_stopped_by_phone(void) { + prv_start_stream_ok(); + mic_capture_service_handle_stream_setup_result(VoiceEndpointResultSuccess); + cl_assert(s_mic_running); + + s_transfer_stop_cb(s_transfer_session); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + // The endpoint already tore its side down; we must not send a stop back + cl_assert_equal_i(0, s_transfer_stopped_count); + cl_assert_equal_i(0, s_transfer_cancelled_count); + prv_assert_stopped_event(MicCaptureStopReasonPhone); +} + +void test_mic_capture_service__stream_preempted_by_system(void) { + prv_start_stream_ok(); + mic_capture_service_handle_stream_setup_result(VoiceEndpointResultSuccess); + + mic_capture_service_handle_system_preempt(); + cl_assert(!mic_capture_service_is_active()); + cl_assert(!s_mic_running); + cl_assert_equal_i(MicClientNone, mic_manager_get_owner()); + cl_assert_equal_i(1, s_transfer_stopped_count); + cl_assert_equal_i(1, s_encoder_close_count); + prv_assert_stopped_event(MicCaptureStopReasonPreempted); +} + +void test_mic_capture_service__stream_refusals(void) { + s_permission_state = AppPermissionStateDenied; + cl_assert_equal_i(MicCaptureStartErrDenied, mic_capture_service_start_stream(PebbleTask_App)); + s_permission_state = AppPermissionStateGranted; + + cl_assert(prv_start_dictation()); + cl_assert_equal_i(MicCaptureStartErrBusy, mic_capture_service_start_stream(PebbleTask_App)); + mic_manager_release(MicClientVoiceDictation); + + // Capture and stream are exclusive + cl_assert_equal_i(MicCaptureStartOk, mic_capture_service_start(PebbleTask_App, SPU)); + cl_assert_equal_i(MicCaptureStartErrBusy, mic_capture_service_start_stream(PebbleTask_App)); + mic_capture_service_stop(PebbleTask_App); + cl_assert_equal_i(0, s_setup_sessions); + + // System apps stream without a UUID tag + s_md.is_unprivileged = false; + prv_start_stream_ok(); + cl_assert(!s_setup_had_uuid); +} diff --git a/tools/generate_native_sdk/exported_symbols.json b/tools/generate_native_sdk/exported_symbols.json index 521cc79612..412c7602de 100644 --- a/tools/generate_native_sdk/exported_symbols.json +++ b/tools/generate_native_sdk/exported_symbols.json @@ -4,7 +4,7 @@ "You should also make sure you are obeying our API design guidelines:", "https://pebbletechnology.atlassian.net/wiki/display/DEV/SDK+API+Design+Guidelines" ], - "revision" : "112", + "revision" : "113", "version" : "2.0", "files": [ "include/pbl/drivers/ambient_light.h", @@ -4826,6 +4826,24 @@ "type": "function", "name": "mic_data_service_is_active", "addedRevision": "111" + }, { + "type": "type", + "name": "MicStreamStartedHandler" + }, { + "type": "type", + "name": "MicStreamHandlers" + }, { + "type": "function", + "name": "mic_stream_to_phone_start", + "addedRevision": "113" + }, { + "type": "function", + "name": "mic_stream_to_phone_stop", + "addedRevision": "113" + }, { + "type": "function", + "name": "mic_stream_to_phone_is_active", + "addedRevision": "113" } ] }, {