fix: preserve Notes after incomplete loads

Author: DevanMetz

.github/workflows/release.yml

@@ -31,6 +31,8 @@ jobs:
           "$RUNNER_TEMP/storage_io_test"
       - name: Test CSV tail recovery
         run: python3 tools/test_storage_csv_tail.py --compiler cc
+      - name: Test Notes storage
+        run: python3 tools/test_notes_io.py --compiler cc
       - name: Test capture reader
         run: |
           cc -std=c11 -Wall -Wextra -Werror -pedantic -I main main/capture_data.c tests/capture_data_test.c -o "$RUNNER_TEMP/capture_data_test"
@@ -114,6 +116,21 @@ jobs:
           if-no-files-found: warn
           retention-days: 14
 
+      - name: Retain Notes storage test evidence
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: notes-io-logs
+          path: |
+            build/notes-io/compile.log
+            build/notes-io/result.log
+            build/notes-io/source.json
+            build/notes-io/notes_callbacks.inc
+            build/notes-io/notes_config.inc
+            build/notes-io/storage_source.inc
+          if-no-files-found: warn
+          retention-days: 14
+
   firmware:
     runs-on: ubuntu-latest
     container: espressif/idf:v5.4.2
@@ -219,6 +236,23 @@ jobs:
       - name: Test offline apps, network consoles and OTA manifests/images
         shell: pwsh
         run: ./tools/test_native_ui_ci.ps1
+      - name: Test Notes with real LVGL
+        shell: pwsh
+        run: python tools/test_notes_io.py --compiler clang --lvgl-library build/offline-ui/out/lvgl_host.lib --output build/notes-ui
+      - name: Retain Notes UI test evidence
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: notes-ui-logs
+          path: |
+            build/notes-ui/compile.log
+            build/notes-ui/result.log
+            build/notes-ui/source.json
+            build/notes-ui/notes_callbacks.inc
+            build/notes-ui/notes_config.inc
+            build/notes-ui/storage_source.inc
+          if-no-files-found: warn
+          retention-days: 14
       - uses: actions/upload-artifact@v4
         if: always()
         with:

ROADMAP.md

@@ -62,6 +62,7 @@ Phase 1 software progress is also underway:
 
 - [x] System reports the firmware/commit, reset reason, uptime, panel, heap, storage capacity, current OTA state, and persistent last OTA result.
 - [x] Notes and ride summaries use recoverable backup replacement; rides publish only after durable sync, and health logs repair incomplete final rows.
+- [x] Notes loads complete files up to 65,535 bytes, replacing a silent 2,047-byte prefix. Recovery/read/close errors, oversized files and NUL data disable Save; byte-length and SD checks run before storage changes. Its launcher leave hook clears editor ownership. Four preceding-source controls reproduce full-note truncation; 36 native file/fault cases and five real LVGL cases pass, including maximum ASCII/UTF-8 notes and 25 reopens without retained resources in the unchanged 96 KiB pool. Linux and Windows CI retain separate source-bound evidence. Physical SD/panel and power cuts remain open.
 - [x] Shared CSV tail recovery now scans backwards in 512-byte blocks. Eight previous-source controls exceed stdio seek budgets; a 64 KiB partial tail drops from 65,538 seeks to 130 with the same retained prefix. All 32 native byte/boundary/fault cases, original storage checks and five MQTT/HTTP logging cases pass with closed modeled handles. The unflashed firmware builds with unchanged configuration. Linux CI retains source hashes and compilation/result logs; physical SD timing and power cuts remain open.
 - [x] Built-in SD paths, timestamps, units, 8.3-safe collision names, `.TMP`, and `.BAK` behavior are documented.
 - [x] Persisted brightness choices, a dimmed two-minute screen, and configurable 5/10/30-minute or Never screen-off behavior are implemented.

docs/architecture.md

@@ -83,6 +83,8 @@ The serial log viewer loads at most 512 validated rows and closes the file befor
 
 CSV tail repair uses a 512-byte scratch buffer and scans backwards by blocks before truncating at the most recent LF. HTTP, MQTT and Ring health logging share this helper with serialized writers. A seek/read failure prevents truncation; later truncate/sync/close failures preserve the first error. Native checks verify retained bytes and reduced stdio call counts, while physical FAT/SD latency and power-loss durability remain device checks.
 
+Notes uses a temporary PSRAM buffer to load an entire file of up to 65,535 bytes and frees that buffer after LVGL copies the text. It applies the editor's character limit after loading, using LVGL's bulk text path. Recovery/read/close failures, oversized files and NUL data disable editing and Save; the save callback rechecks ownership, SD availability and byte length before changing paths. The launcher leave hook clears its editor pointer and permission before content deletion. The existing replacement helper still syncs/closes `NOTE.TMP`, retains the previous complete generation in `NOTE.BAK`, and reports publication failures. Real LVGL checks use the unchanged 96 KiB pool; physical SD and panel behavior remain open.
+
 See [Storage and compatibility contract](compatibility.md) and [Data formats](data-formats.md).
 
 ## Network and radio

docs/data-formats.md

@@ -21,6 +21,8 @@ Tab5 OS keeps user-owned documents and logs on the removable microSD card. Built
 
 Files ending in `.TMP` are unpublished: they may be incomplete after power loss, or fully synced but retained because publication failed. They must not be reported as finished captures without validation. A `.BAK` file is the previous complete generation and can be used for recovery when its corresponding final file is absent.
 
+Notes edits complete text files of at most 65,535 bytes. A missing note starts an empty draft; a missing final note can be restored from `NOTE.BAK`. Recovery, open, read or close failures disable editing and Save until a successful reopen. Oversized files and files containing NUL bytes also disable Save and keep their original bytes. Save checks the draft's byte length before changing any path, including for UTF-8 text whose character count is smaller than its byte count. Notes loads the complete text in bulk before applying the editor's character limit. Leaving Notes clears its editor pointer and Save permission. Native files and LVGL checks verify these boundaries; physical SD removal, full-card and power-loss behavior remain hardware gates.
+
 Files opens published Scope and I2C `.CSV` captures as graphs when their headers match the formats below. The viewer shows up to the first 2,048 well-formed rows, leaves the file unchanged, and shows failed I2C reads as gaps. Other files retain the text preview.
 
 Files also opens published UART and RS-485 `.CSV` logs as a timestamped RX/TX timeline. The viewer loads up to the first 512 well-formed rows, shows eight rows per page, and lets you filter by direction or switch between hex and escaped ASCII without changing the file. Unpublished `.TMP` files retain the text preview.

docs/testing.md

@@ -11,6 +11,7 @@ Python 3 and Node.js are needed for the quick checks. The configuration checks a
 | SDK setup, board selection or root CMake | [Firmware configuration](#firmware-configuration). |
 | Relay, parsers, calculations, clipboard or file formats | [Core command reference](#core-command-reference); choose the affected module and any independent reference test. |
 | Storage writes or incomplete CSV rows | [Core command reference](#core-command-reference), [CSV tail recovery](#csv-tail-recovery) and a firmware build. |
+| Notes loading, saving or lifecycle | [Notes storage and UI](#notes-storage-and-ui) and a firmware build. |
 | Offline apps, SPI or I2C editor/result flows | [Offline and peripheral UI](#offline-and-peripheral-ui). |
 | Serial, saved serial logs or RTU Frames | [Serial and RTU UI](#serial-and-rtu-ui). |
 | Modbus TCP, NTP, Wake-on-LAN, UDP or HTTP fixtures | [Network fixtures and UI](#network-fixtures-and-ui). |
@@ -96,6 +97,20 @@ Seek/read failures preserve all bytes, including a short read without a supplied
 
 Generated source, full storage/header/test hashes and compile/result logs stay under `build/storage-csv-tail/`. Linux CI runs the fixture and retains those files as `storage-csv-tail-logs`, including compiler/test failures. Its 32 cases are separate from the seven-suite Windows native total.
 
+## Notes storage and UI
+
+```powershell
+python tools/test_notes_io.py --compiler 'C:/path/to/clang.exe'
+# With the cached Debug LVGL library from the offline UI runner:
+python tools/test_notes_io.py --compiler 'C:/path/to/clang.exe' --lvgl-library build/offline-ui/out/lvgl_host.lib --output build/notes-ui
+```
+
+The portable runner compiles four unchanged Notes functions extracted from `main/main.c` and the actual complete `storage_io.c`. Its 36 native-file cases cover missing/empty notes, backup recovery, the former 2 KiB boundary, 5 KiB and maximum-size files, oversized and NUL-containing files, allocation/recovery/open/read/close errors, the first read error, save length/UTF-8 guards, card availability, stream/publication/rollback failures and stale Save/reopen behavior. Every case independently checks final, backup and temporary bytes and closes its modeled file/allocation handles. Four prior-source full-file controls reproduce truncation to 2,047 bytes; recovery and read faults also reproduce unsafe Save actions. New allocation/probe/leave controls test the added boundaries rather than asserting that the previous callbacks used those APIs.
+
+The optional real LVGL mode runs five additional cases against the cached pinned library with its existing 96 KiB pool. It checks complete 65,535-byte ASCII and 65,534-byte UTF-8 load/save, an oversized original file, a multibyte draft whose byte count exceeds the limit, and 25 reopen cycles with identical free bytes and allocation counts. Text is installed before the character limit so LVGL uses its bulk path. Files are native; SD faults and PSRAM allocation APIs are controlled. This does not prove physical rendering, card latency, removal or power-loss durability.
+
+Generated callbacks/config/storage source, full main/function/storage/header/test hashes and compile/result logs stay under the selected output directory. Linux CI retains the 36-case evidence as `notes-io-logs`. Windows CI runs the five real LVGL cases after the existing seven suites and retains separate `notes-ui-logs`; those five add to the prior 815 native checks. Both artifacts retain diagnostics when their test step fails.
+
 ## OTA manifests
 
 ```powershell

main/main.c

@@ -130,6 +130,7 @@
 #define BROWSER_MAX_TEXT 12288
 #define BROWSER_MAX_LINKS 12
 #define EBOOK_PAGE_BYTES 8192
+#define NOTE_MAX_BYTES 65535
 #define OTA_MANIFEST_URL "https://github.com/DevanMetz/Tab5OS/releases/latest/download/tab5_os.json"
 #define BATTERY_EMPTY_MV 6000
 #define BATTERY_FULL_MV 8230
@@ -359,6 +360,7 @@ static int battery_millivolts;
 static int battery_milliamps;
 static int battery_percent;
 static lv_obj_t *note_area;
+static bool note_can_save;
 static lv_obj_t *counter_label;
 static bool internal_ready;
 static esp_err_t storage_init_error = ESP_OK;
@@ -4563,9 +4565,44 @@ static void ebooks_clicked(lv_event_t *event)
     show_ebooks();
 }
 
+static void notes_leave(void)
+{
+    note_area = NULL;
+    note_can_save = false;
+}
+
+static int load_note(char *note)
+{
+    note[0] = '\0';
+    if (storage_recover_replace(SD_PATH "/DOCS/NOTE.TXT", SD_PATH "/DOCS/NOTE.BAK") != 0)
+        return errno ? errno : EIO;
+    FILE *file = fopen(SD_PATH "/DOCS/NOTE.TXT", "rb");
+    if (!file) return errno == ENOENT ? 0 : errno ? errno : EIO;
+
+    errno = 0;
+    size_t read = fread(note, 1, NOTE_MAX_BYTES, file);
+    int error = ferror(file) ? (errno ? errno : EIO) : 0;
+    if (!error && fgetc(file) != EOF) error = EFBIG;
+    if (!error && ferror(file)) error = errno ? errno : EIO;
+    if (!error && memchr(note, '\0', read)) error = EILSEQ;
+    if (fclose(file) != 0 && !error) error = errno ? errno : EIO;
+    note[read] = '\0';
+    return error;
+}
+
 static void save_note(lv_event_t *event)
 {
+    if (!note_can_save || !note_area) return;
     lv_obj_t *status = lv_event_get_user_data(event);
+    if (!sd_ready) {
+        lv_label_set_text(status, "Save failed: SD card unavailable");
+        return;
+    }
+    const char *text = lv_textarea_get_text(note_area);
+    if (strlen(text) > NOTE_MAX_BYTES) {
+        lv_label_set_text_fmt(status, "Note too long (maximum %d bytes)", NOTE_MAX_BYTES);
+        return;
+    }
     mkdir(SD_PATH "/DOCS", 0775);
     const char *temporary_path = SD_PATH "/DOCS/NOTE.TMP";
     const char *final_path = SD_PATH "/DOCS/NOTE.TXT";
@@ -4578,7 +4615,7 @@ static void save_note(lv_event_t *event)
         lv_label_set_text_fmt(status, "Save failed: %s", strerror(error));
         return;
     }
-    if (fputs(lv_textarea_get_text(note_area), file) < 0) {
+    if (fputs(text, file) < 0) {
         int write_error = errno ? errno : EIO;
         fclose(file);
         file = NULL;
@@ -4617,21 +4654,36 @@ static void notes_clicked(lv_event_t *event)
     lv_obj_t *save_label = lv_label_create(save);
     lv_label_set_text(save_label, "Save");
     lv_obj_center(save_label);
-    if (!sd_ready) lv_obj_add_state(save, LV_STATE_DISABLED);
-
-    static char note[2048];
-    storage_recover_replace(SD_PATH "/DOCS/NOTE.TXT", SD_PATH "/DOCS/NOTE.BAK");
-    FILE *file = fopen(SD_PATH "/DOCS/NOTE.TXT", "rb");
-    size_t read = file ? fread(note, 1, sizeof(note) - 1, file) : 0;
-    if (file) fclose(file);
-    note[read] = '\0';
+    note_can_save = false;
+    char *note = sd_ready ? heap_caps_malloc(NOTE_MAX_BYTES + 1, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT) : NULL;
+    if (sd_ready && !note) {
+        lv_label_set_text(status, "Load failed: not enough memory; reopen Notes to retry");
+    } else if (note) {
+        int error = load_note(note);
+        if (!error) note_can_save = true;
+        else if (error == EFBIG)
+            lv_label_set_text_fmt(status, "Note exceeds %d bytes; Save disabled", NOTE_MAX_BYTES);
+        else if (error == EILSEQ)
+            lv_label_set_text(status, "Note contains NUL bytes; Save disabled");
+        else {
+            sd_record_error(error);
+            lv_label_set_text_fmt(status, "Load failed: %s; reopen Notes to retry", strerror(error));
+        }
+    }
 
     note_area = lv_textarea_create(content);
     lv_obj_set_size(note_area, 640, 420);
-    lv_textarea_set_text(note_area, note);
+    lv_textarea_set_text(note_area, note_can_save ? note : "");
+    lv_textarea_set_max_length(note_area, NOTE_MAX_BYTES);
+    heap_caps_free(note);
     lv_obj_t *keyboard = lv_keyboard_create(content);
     lv_obj_set_size(keyboard, 640, 500);
     lv_keyboard_set_textarea(keyboard, note_area);
+    if (!note_can_save) {
+        lv_obj_add_state(save, LV_STATE_DISABLED);
+        lv_obj_add_state(note_area, LV_STATE_DISABLED);
+        lv_obj_add_state(keyboard, LV_STATE_DISABLED);
+    }
 }
 
 static void update_counter(void)
@@ -7597,7 +7649,7 @@ static void screensaver_tick(lv_timer_t *timer)
 
 static const app_definition_t launcher_apps[] = {
     {LV_SYMBOL_DIRECTORY, "Files", 0x2196F3, files_clicked, NULL, 0, 0},
-    {LV_SYMBOL_EDIT, "Notes", 0x00A896, notes_clicked, NULL, 1, 0},
+    {LV_SYMBOL_EDIT, "Notes", 0x00A896, notes_clicked, notes_leave, 1, 0},
     {LV_SYMBOL_PLUS, "Counter", 0xF59E0B, counter_clicked, NULL, 2, 0},
     {LV_SYMBOL_CHARGE, "GPIO", 0xE65100, gpio_clicked, NULL, 3, 0},
     {LV_SYMBOL_WIFI, "Settings", 0x0288D1, settings_clicked, settings_leave, 0, 1},