From 2dab90b149cf05b8e67c8c41ae8f978f6c3b59f8 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 14 Oct 2026 20:16:35 -0600 Subject: [PATCH 01/17] Fix "local track published for unknown sid" warning --- src/ffi_client.cpp | 19 ++++++++--- src/ffi_client.h | 6 ++-- src/local_participant.cpp | 27 +++++++-------- .../test_local_track_publish_sid.cpp | 33 +++++++++++++++++++ 4 files changed, 62 insertions(+), 23 deletions(-) diff --git a/src/ffi_client.cpp b/src/ffi_client.cpp index 867eba94..2a3eba73 100644 --- a/src/ffi_client.cpp +++ b/src/ffi_client.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -694,9 +695,9 @@ std::future FfiClient::simulateScenarioAsync(uintptr_t room_handle, int sc } // Participant APIs Implementation -std::future FfiClient::publishTrackAsync(std::uint64_t local_participant_handle, - std::uint64_t track_handle, - const TrackPublishOptions& options) { +std::future FfiClient::publishTrackAsync( + std::uint64_t local_participant_handle, std::uint64_t track_handle, const TrackPublishOptions& options, + std::function on_success) { // Generate client-side async_id first const AsyncId async_id = generateAsyncId(); @@ -708,7 +709,8 @@ std::future FfiClient::publishTrackAsync(std::uint return event.has_publish_track() && event.publish_track().async_id() == async_id; }, // Handler: resolve with publication or throw error - [](const proto::FfiEvent& event, std::promise& pr) { + [on_success = std::move(on_success)](const proto::FfiEvent& event, + std::promise& pr) { const auto& cb = event.publish_track(); // Oneof message { string error = 2; OwnedTrackPublication publication = @@ -723,7 +725,14 @@ std::future FfiClient::publishTrackAsync(std::uint } const proto::OwnedTrackPublication& pub = cb.publication(); - pr.set_value(pub); + try { + if (on_success) { + on_success(pub); + } + pr.set_value(pub); + } catch (...) { + pr.set_exception(std::current_exception()); + } }); // Build and send the request diff --git a/src/ffi_client.h b/src/ffi_client.h index 0104de76..4f10b1fa 100644 --- a/src/ffi_client.h +++ b/src/ffi_client.h @@ -109,9 +109,9 @@ class LIVEKIT_INTERNAL_API FfiClient { std::future simulateScenarioAsync(uintptr_t room_handle, int scenario); // Participant APIs - std::future publishTrackAsync(std::uint64_t local_participant_handle, - std::uint64_t track_handle, - const TrackPublishOptions& options); + std::future publishTrackAsync( + std::uint64_t local_participant_handle, std::uint64_t track_handle, const TrackPublishOptions& options, + std::function on_success = {}); std::future unpublishTrackAsync(std::uint64_t local_participant_handle, const std::string& track_sid, bool stop_on_unpublish); std::future publishDataAsync(std::uint64_t local_participant_handle, const std::uint8_t* data_ptr, diff --git a/src/local_participant.cpp b/src/local_participant.cpp index 8dbc1fdc..2607963d 100644 --- a/src/local_participant.cpp +++ b/src/local_participant.cpp @@ -191,21 +191,18 @@ void LocalParticipant::publishTrack(const std::shared_ptr& track, const T if (track_handle == 0) { throw std::runtime_error("LocalParticipant::publishTrack: invalid track FFI handle"); } - auto fut = FfiClient::instance().publishTrackAsync(static_cast(participant_handle), - static_cast(track_handle), options); - - // Will throw if the async op fails (error in callback). - const proto::OwnedTrackPublication owned_pub = fut.get(); - - // Construct a LocalTrackPublication from the proto publication. - auto publication = std::make_shared(owned_pub); - - const std::string sid = publication->sid(); - { - const std::scoped_lock guard(published_tracks_mutex_); - published_tracks_by_sid_[sid] = std::weak_ptr(track); - track->setPublication(publication); - } + auto fut = FfiClient::instance().publishTrackAsync( + static_cast(participant_handle), static_cast(track_handle), options, + [this, track](const proto::OwnedTrackPublication& owned_pub) { + auto publication = std::make_shared(owned_pub); + const std::scoped_lock guard(published_tracks_mutex_); + published_tracks_by_sid_[publication->sid()] = track; + track->setPublication(publication); + }); + + // Publication state is installed on the FFI thread before this future is + // completed and before Rust can emit LocalTrackPublished. + (void)fut.get(); } std::shared_ptr LocalParticipant::publishVideoTrack(const std::string& name, diff --git a/src/tests/integration/test_local_track_publish_sid.cpp b/src/tests/integration/test_local_track_publish_sid.cpp index ee767450..21707f4b 100644 --- a/src/tests/integration/test_local_track_publish_sid.cpp +++ b/src/tests/integration/test_local_track_publish_sid.cpp @@ -17,9 +17,12 @@ #include #include +#include #include +#include #include #include +#include #include "../common/audio_utils.h" #include "../common/room_test_access.h" @@ -31,6 +34,30 @@ namespace { using namespace std::chrono_literals; +class LocalPublicationDelegate final : public RoomDelegate { +public: + void onLocalTrackPublished(Room&, const LocalTrackPublishedEvent& event) override { + if (!event.publication) { + return; + } + { + const std::scoped_lock guard(mutex_); + published_sids_.insert(event.publication->sid()); + } + cv_.notify_all(); + } + + bool waitForPublished(const std::string& sid, std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return cv_.wait_for(lock, timeout, [&] { return published_sids_.count(sid) != 0; }); + } + +private: + std::mutex mutex_; + std::condition_variable cv_; + std::unordered_set published_sids_; +}; + void expectTrackSidAssigned(const Track& track, const LocalTrackPublication& publication) { const std::string& track_sid = track.sid(); const std::string& publication_sid = publication.sid(); @@ -58,6 +85,8 @@ TEST_F(LocalTrackPublishSidTest, PublishVideoTrackAssignsSid) { failIfNotConfigured(); Room room; + LocalPublicationDelegate delegate; + room.setDelegate(&delegate); const RoomOptions room_options; ASSERT_TRUE(room.connect(config_.url, config_.token_a, room_options)); @@ -69,6 +98,7 @@ TEST_F(LocalTrackPublishSidTest, PublishVideoTrackAssignsSid) { ASSERT_NE(track->publication(), nullptr); expectTrackSidAssigned(*track, *track->publication()); + EXPECT_TRUE(delegate.waitForPublished(track->sid(), 5s)); lockLocalParticipant(room)->unpublishTrack(track->publication()->sid()); } @@ -76,6 +106,8 @@ TEST_F(LocalTrackPublishSidTest, PublishAudioTrackAssignsSid) { failIfNotConfigured(); Room room; + LocalPublicationDelegate delegate; + room.setDelegate(&delegate); const RoomOptions room_options; ASSERT_TRUE(room.connect(config_.url, config_.token_a, room_options)); @@ -87,6 +119,7 @@ TEST_F(LocalTrackPublishSidTest, PublishAudioTrackAssignsSid) { ASSERT_NE(track->publication(), nullptr); expectTrackSidAssigned(*track, *track->publication()); + EXPECT_TRUE(delegate.waitForPublished(track->sid(), 5s)); lockLocalParticipant(room)->unpublishTrack(track->publication()->sid()); } From 5adaeef0c0cbe5b0a29dcdc495751ac04b0dec17 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 17:55:31 -0700 Subject: [PATCH 02/17] Memory regression test --- .github/workflows/ci.yml | 1 + .github/workflows/tests.yml | 15 ++ docs/testing.md | 26 +++ scripts/memory-regression.sh | 95 +++++++++++ src/tests/CMakeLists.txt | 19 +++ src/tests/memory/source_lifecycle_tester.cpp | 160 +++++++++++++++++++ 6 files changed, 316 insertions(+) create mode 100755 scripts/memory-regression.sh create mode 100644 src/tests/memory/source_lifecycle_tester.cpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c20ff88..87152589 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,6 +55,7 @@ jobs: - client-sdk-rust - client-sdk-rust/** - cmake/** + - scripts/memory-regression.sh - scripts/set-test-tokens.sh - scripts/run-with-backtrace.sh - CMakeLists.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index efeaf448..c38a0a69 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -425,6 +425,21 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml + - name: Run Linux memory regression gate + if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' + timeout-minutes: 10 + shell: bash + env: + MALLOC_ARENA_MAX: "1" + run: | + set -euo pipefail + scripts/memory-regression.sh \ + --build-dir "${{ env.BUILD_DIR }}" \ + --iterations 100 \ + --warmup 20 \ + --max-rss-growth-kib 8192 \ + --max-thread-growth 0 + - name: Run stress tests if: matrix.e2e-testing && inputs.run_stress_tests timeout-minutes: ${{ inputs.stress_timeout_minutes }} diff --git a/docs/testing.md b/docs/testing.md index 33919c0e..383dc48a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -106,6 +106,32 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. +## Linux memory regression gate + +The Linux PR gate repeatedly initializes the SDK, creates and destroys unused +audio/video sources and tracks, then shuts the SDK down. It compares process +RSS and thread count after cycle 20 with cycle 100, failing on thread growth or +more than 8 MiB of RSS growth. + +The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits +glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a +controlled CI signal, not evidence that the default allocator returns all +freed pages to the OS. + +To reproduce locally (no LiveKit server or tokens are required): + +```bash +./build.sh release-tests +scripts/memory-regression.sh \ + --iterations 100 --warmup 20 \ + --max-rss-growth-kib 8192 --max-thread-growth 0 +``` + +The limits are command-line options so a longer local run can use different +budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends a compact +results table when `GITHUB_STEP_SUMMARY` is set. Hardware encoding and +connected-room lifecycle coverage are separate follow-up work. + ## Memory checks (valgrind) Run `valgrind` against the test binaries to check for memory leaks and other diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh new file mode 100755 index 00000000..c3f217d4 --- /dev/null +++ b/scripts/memory-regression.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Copyright 2026 LiveKit +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: memory-regression.sh [OPTIONS] + +Run the Linux memory lifecycle regression tester and append a table to the +GitHub step summary when GITHUB_STEP_SUMMARY is set. + +Options: + --build-dir PATH Build directory (default: build-release) + --iterations N Lifecycle cycles (default: 100) + --warmup N Baseline cycle (default: 20) + --max-rss-growth-kib N Allowed RSS growth (default: 8192) + --max-thread-growth N Allowed thread growth (default: 0) + -h, --help Show this help + +Environment: + MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_ITERATIONS, + MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, + MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX +EOF +} + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" +iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" +warmup="${MEMORY_REGRESSION_WARMUP:-20}" +max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" +max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" + +while (($#)); do + case "$1" in + --build-dir) build_dir="$2"; shift 2 ;; + --iterations) iterations="$2"; shift 2 ;; + --warmup) warmup="$2"; shift 2 ;; + --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; + --max-thread-growth) max_thread_growth="$2"; shift 2 ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;; + esac +done + +if [[ "${build_dir}" != /* ]]; then + build_dir="${repo_root}/${build_dir}" +fi +tester="${build_dir}/bin/livekit_memory_source_lifecycle_tester" +if [[ ! -x "${tester}" ]]; then + echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 + exit 1 +fi + +set +e +output="$( + MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ + --iterations "${iterations}" \ + --warmup "${warmup}" \ + --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-thread-growth "${max_thread_growth}" 2>&1 +)" +status=$? +set -e +printf '%s\n' "${output}" + +report_regex='memory lifecycle: RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +if [[ -n "${GITHUB_STEP_SUMMARY:-}" && "${output}" =~ ${report_regex} ]]; then + { + echo "## Linux memory regression" + echo + echo "| Metric | Baseline | Final | Growth | Limit |" + echo "|---|---:|---:|---:|---:|" + echo "| RSS | ${BASH_REMATCH[1]} KiB | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${max_rss_growth_kib} KiB |" + echo "| Threads | ${BASH_REMATCH[4]} | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${max_thread_growth} |" + echo + echo "**Verdict: ${BASH_REMATCH[7]}**" + } >> "${GITHUB_STEP_SUMMARY}" +fi + +exit "${status}" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 9583af73..c9080662 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -348,3 +348,22 @@ endif() if(TARGET livekit_stress_tests) add_dependencies(run_all_tests livekit_stress_tests) endif() + +if(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_executable(livekit_memory_source_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/memory/source_lifecycle_tester.cpp" + ) + target_link_libraries(livekit_memory_source_lifecycle_tester PRIVATE livekit) + target_include_directories(livekit_memory_source_lifecycle_tester PRIVATE + ${LIVEKIT_ROOT_DIR}/include + ) + add_custom_command(TARGET livekit_memory_source_lifecycle_tester POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ + $ + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$/liblivekit_ffi.so" + $ + COMMENT "Copying shared libraries next to livekit_memory_source_lifecycle_tester" + ) +endif() diff --git a/src/tests/memory/source_lifecycle_tester.cpp b/src/tests/memory/source_lifecycle_tester.cpp new file mode 100644 index 00000000..197da1ee --- /dev/null +++ b/src/tests/memory/source_lifecycle_tester.cpp @@ -0,0 +1,160 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +struct Options { + std::uint64_t iterations{100}; + std::uint64_t warmup{20}; + std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_thread_growth{0}; +}; + +struct ProcessSample { + std::uint64_t rss_kib{}; + std::uint64_t threads{}; +}; + +std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { + std::uint64_t parsed = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + throw std::runtime_error("invalid value for " + std::string(option)); + } + return parsed; +} + +Options parseOptions(int argc, char* argv[]) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string_view option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + std::string(option)); + } + const std::string_view value(argv[++index]); + if (option == "--iterations") { + options.iterations = parseUnsigned(value, option); + } else if (option == "--warmup") { + options.warmup = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-kib") { + options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-thread-growth") { + options.max_thread_growth = parseUnsigned(value, option); + } else { + throw std::runtime_error("unknown option: " + std::string(option)); + } + } + if (options.warmup == 0 || options.warmup >= options.iterations) { + throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); + } + return options; +} + +std::uint64_t statusValue(std::string_view line) { + const auto separator = line.find(':'); + if (separator == std::string_view::npos) { + throw std::runtime_error("malformed /proc/self/status value"); + } + line.remove_prefix(separator + 1); + while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { + line.remove_prefix(1); + } + const auto end = line.find_first_not_of("0123456789"); + return parseUnsigned(line.substr(0, end), "/proc/self/status"); +} + +ProcessSample sampleProcess() { + ProcessSample sample; + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmRSS:", 0) == 0) { + sample.rss_kib = statusValue(line); + } else if (line.rfind("Threads:", 0) == 0) { + sample.threads = statusValue(line); + } + } + if (sample.rss_kib == 0 || sample.threads == 0) { + throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); + } + return sample; +} + +void runCycle() { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("LiveKit initialization failed"); + } + try { + { + auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!audio_track || !video_track) { + throw std::runtime_error("failed to create local tracks"); + } + } + } catch (...) { + livekit::shutdown(); + throw; + } + livekit::shutdown(); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + const Options options = parseOptions(argc, argv); + ProcessSample baseline; + ProcessSample final; + for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { + runCycle(); + if (iteration == options.warmup) { + baseline = sampleProcess(); + } else if (iteration == options.iterations) { + final = sampleProcess(); + } + } + + const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); + const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); + const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && + thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: RSS " << baseline.rss_kib << " -> " << final.rss_kib << " KiB (" << rss_growth + << " KiB), threads " << baseline.threads << " -> " << final.threads << " (" << thread_growth + << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } +} From a89e5ebaa328f8a5971c8a38b468797e263e84ef Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:01:44 -0700 Subject: [PATCH 03/17] More scenarios --- .github/workflows/tests.yml | 14 +- docs/testing.md | 35 +- scripts/memory-regression.sh | 92 +++-- src/tests/CMakeLists.txt | 16 +- src/tests/memory/lifecycle_tester.cpp | 334 +++++++++++++++++++ src/tests/memory/source_lifecycle_tester.cpp | 160 --------- 6 files changed, 451 insertions(+), 200 deletions(-) create mode 100644 src/tests/memory/lifecycle_tester.cpp delete mode 100644 src/tests/memory/source_lifecycle_tester.cpp diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c38a0a69..9accd92d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -357,7 +357,11 @@ jobs: --gtest_output="xml:${{ env.BUILD_DIR }}\unit-test-results.xml" - name: Start livekit-server - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: >- + matrix.e2e-testing && + (inputs.integration_repeat > 0 || + inputs.run_stress_tests || + (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release')) id: livekit_server uses: livekit/dev-server-action@5d4d5337a875e2d1afd37bed03c601d159dab002 # v1.1.1 with: @@ -368,7 +372,11 @@ jobs: # Needed by token helper script - name: Install livekit-cli - if: matrix.e2e-testing && (inputs.integration_repeat > 0 || inputs.run_stress_tests) + if: >- + matrix.e2e-testing && + (inputs.integration_repeat > 0 || + inputs.run_stress_tests || + (matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release')) shell: bash env: # Windows installs lk via `gh api` / `gh release download`, which need this env var @@ -433,8 +441,10 @@ jobs: MALLOC_ARENA_MAX: "1" run: | set -euo pipefail + source scripts/set-test-tokens.sh scripts/memory-regression.sh \ --build-dir "${{ env.BUILD_DIR }}" \ + --scenario all \ --iterations 100 \ --warmup 20 \ --max-rss-growth-kib 8192 \ diff --git a/docs/testing.md b/docs/testing.md index 383dc48a..90ebb4d5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -44,6 +44,7 @@ __Note:__ The tests require tokens and a running LiveKit server. See the section | `livekit_unit_tests` | Pure unit tests (no server required) | | `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality | | `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) | +| `livekit_memory_lifecycle_tester` | Linux lifecycle RSS and thread regression scenarios | ## Running a local LiveKit server for tests @@ -108,29 +109,47 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate repeatedly initializes the SDK, creates and destroys unused -audio/video sources and tracks, then shuts the SDK down. It compares process -RSS and thread count after cycle 20 with cycle 100, failing on thread growth or -more than 8 MiB of RSS growth. +The Linux PR gate runs three lifecycle scenarios in separate processes: + +- `sdk-sources` repeatedly initializes the SDK, creates and destroys unused + audio/video sources and tracks, then shuts the SDK down. +- `room-client-leave` repeatedly connects, disconnects, and destroys a room. +- `room-server-delete` repeatedly connects and uses `lk` to delete the local + test room, then verifies the disconnect and end-of-stream callbacks. + +Each scenario compares process RSS and thread count after cycle 20 with cycle +100, failing on thread growth or more than 8 MiB of RSS growth. The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -To reproduce locally (no LiveKit server or tokens are required): +The offline source scenario requires no LiveKit server or tokens: ```bash ./build.sh release-tests scripts/memory-regression.sh \ + --scenario sdk-sources \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` +To run either room scenario, start `livekit-server --dev`, install the `lk` +CLI, and source the local test tokens. Use `--scenario all` to run all three +scenarios, each in a fresh tester process: + +```bash +source scripts/set-test-tokens.sh +scripts/memory-regression.sh --scenario room-client-leave +scripts/memory-regression.sh --scenario room-server-delete +scripts/memory-regression.sh --scenario all +``` + The limits are command-line options so a longer local run can use different -budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends a compact -results table when `GITHUB_STEP_SUMMARY` is set. Hardware encoding and -connected-room lifecycle coverage are separate follow-up work. +budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends one results +row per scenario when `GITHUB_STEP_SUMMARY` is set. Hardware encoding coverage +is separate follow-up work. ## Memory checks (valgrind) diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index c3f217d4..1032d769 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,6 +25,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) + --scenario NAME sdk-sources, room-client-leave, + room-server-delete, or all (default: sdk-sources) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -32,14 +34,17 @@ Options: -h, --help Show this help Environment: - MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_ITERATIONS, + MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, + MEMORY_REGRESSION_ITERATIONS, MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, - MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX + MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. + Room scenarios also require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" +scenario="${MEMORY_REGRESSION_SCENARIO:-sdk-sources}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" @@ -48,6 +53,7 @@ max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" while (($#)); do case "$1" in --build-dir) build_dir="$2"; shift 2 ;; + --scenario) scenario="$2"; shift 2 ;; --iterations) iterations="$2"; shift 2 ;; --warmup) warmup="$2"; shift 2 ;; --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; @@ -60,36 +66,78 @@ done if [[ "${build_dir}" != /* ]]; then build_dir="${repo_root}/${build_dir}" fi -tester="${build_dir}/bin/livekit_memory_source_lifecycle_tester" +tester="${build_dir}/bin/livekit_memory_lifecycle_tester" if [[ ! -x "${tester}" ]]; then echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 exit 1 fi -set +e -output="$( - MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ - --iterations "${iterations}" \ - --warmup "${warmup}" \ - --max-rss-growth-kib "${max_rss_growth_kib}" \ - --max-thread-growth "${max_thread_growth}" 2>&1 -)" -status=$? -set -e -printf '%s\n' "${output}" +case "${scenario}" in + sdk-sources|room-client-leave|room-server-delete) + scenarios=("${scenario}") + ;; + all) + scenarios=(sdk-sources room-client-leave room-server-delete) + ;; + *) + echo "ERROR: invalid scenario: ${scenario}" >&2 + usage >&2 + exit 2 + ;; +esac -report_regex='memory lifecycle: RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' -if [[ -n "${GITHUB_STEP_SUMMARY:-}" && "${output}" =~ ${report_regex} ]]; then +for current_scenario in "${scenarios[@]}"; do + if [[ "${current_scenario}" != "sdk-sources" ]] && + [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then + echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 + echo "Run: source scripts/set-test-tokens.sh" >&2 + exit 1 + fi +done + +if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { echo "## Linux memory regression" echo - echo "| Metric | Baseline | Final | Growth | Limit |" - echo "|---|---:|---:|---:|---:|" - echo "| RSS | ${BASH_REMATCH[1]} KiB | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${max_rss_growth_kib} KiB |" - echo "| Threads | ${BASH_REMATCH[4]} | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${max_thread_growth} |" + echo "Limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "**Verdict: ${BASH_REMATCH[7]}**" + echo "| Scenario | RSS baseline | RSS final | RSS growth | Threads baseline | Threads final | Thread growth | Verdict |" + echo "|---|---:|---:|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -exit "${status}" +report_regex='memory lifecycle: scenario=([a-z-]+), RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +overall_status=0 +for current_scenario in "${scenarios[@]}"; do + set +e + output="$( + MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" "${tester}" \ + --scenario "${current_scenario}" \ + --iterations "${iterations}" \ + --warmup "${warmup}" \ + --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-thread-growth "${max_thread_growth}" 2>&1 + )" + status=$? + set -e + printf '%s\n' "${output}" + + if [[ "${output}" =~ ${report_regex} ]]; then + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "| ${BASH_REMATCH[1]} | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} |" \ + >> "${GITHUB_STEP_SUMMARY}" + fi + else + echo "ERROR: ${current_scenario} did not produce a memory report." >&2 + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + echo "| ${current_scenario} | — | — | — | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + fi + status=1 + fi + + if [[ "${status}" -ne 0 ]]; then + overall_status=1 + fi +done + +exit "${overall_status}" diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index c9080662..1204193c 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -350,20 +350,20 @@ if(TARGET livekit_stress_tests) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - add_executable(livekit_memory_source_lifecycle_tester - "${CMAKE_CURRENT_SOURCE_DIR}/memory/source_lifecycle_tester.cpp" + add_executable(livekit_memory_lifecycle_tester + "${CMAKE_CURRENT_SOURCE_DIR}/memory/lifecycle_tester.cpp" ) - target_link_libraries(livekit_memory_source_lifecycle_tester PRIVATE livekit) - target_include_directories(livekit_memory_source_lifecycle_tester PRIVATE + target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) + target_include_directories(livekit_memory_lifecycle_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include ) - add_custom_command(TARGET livekit_memory_source_lifecycle_tester POST_BUILD + add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/liblivekit_ffi.so" - $ - COMMENT "Copying shared libraries next to livekit_memory_source_lifecycle_tester" + $ + COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" ) endif() diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp new file mode 100644 index 00000000..8fbb7535 --- /dev/null +++ b/src/tests/memory/lifecycle_tester.cpp @@ -0,0 +1,334 @@ +/* + * Copyright 2026 LiveKit + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace std::chrono_literals; + +constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; + +enum class Scenario { + SdkSources, + RoomClientLeave, + RoomServerDelete, +}; + +struct Options { + Scenario scenario{Scenario::SdkSources}; + std::uint64_t iterations{100}; + std::uint64_t warmup{20}; + std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_thread_growth{0}; +}; + +struct ProcessSample { + std::uint64_t rss_kib{}; + std::uint64_t threads{}; +}; + +struct RoomCredentials { + std::string url; + std::string token; +}; + +class SdkLifetime { +public: + SdkLifetime() { + if (!livekit::initialize(livekit::LogLevel::Warn)) { + throw std::runtime_error("LiveKit initialization failed"); + } + } + + ~SdkLifetime() { livekit::shutdown(); } + + SdkLifetime(const SdkLifetime&) = delete; + SdkLifetime& operator=(const SdkLifetime&) = delete; +}; + +class DisconnectTrackingDelegate final : public livekit::RoomDelegate { +public: + void onDisconnected(livekit::Room&, const livekit::DisconnectedEvent& event) override { + { + const std::scoped_lock lock(mutex_); + reason_ = event.reason; + ++disconnect_count_; + } + condition_.notify_all(); + } + + void onRoomEos(livekit::Room&, const livekit::RoomEosEvent&) override { + { + const std::scoped_lock lock(mutex_); + ++eos_count_; + } + condition_.notify_all(); + } + + bool waitForTeardown(std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + return condition_.wait_for(lock, timeout, [this]() { return disconnect_count_ > 0 && eos_count_ > 0; }); + } + + void requireDisconnect(livekit::DisconnectReason expected_reason) const { + const std::scoped_lock lock(mutex_); + if (disconnect_count_ != 1 || reason_ != expected_reason) { + throw std::runtime_error("unexpected room disconnect event"); + } + } + + void requireServerTeardown() const { + const std::scoped_lock lock(mutex_); + if (disconnect_count_ != 1 || eos_count_ != 1 || reason_ != livekit::DisconnectReason::RoomDeleted) { + throw std::runtime_error("unexpected server room teardown events"); + } + } + +private: + mutable std::mutex mutex_; + std::condition_variable condition_; + livekit::DisconnectReason reason_{livekit::DisconnectReason::Unknown}; + int disconnect_count_{0}; + int eos_count_{0}; +}; + +std::string_view scenarioName(Scenario scenario) { + switch (scenario) { + case Scenario::SdkSources: + return "sdk-sources"; + case Scenario::RoomClientLeave: + return "room-client-leave"; + case Scenario::RoomServerDelete: + return "room-server-delete"; + } + throw std::runtime_error("unknown scenario"); +} + +Scenario parseScenario(std::string_view value) { + if (value == "sdk-sources") { + return Scenario::SdkSources; + } + if (value == "room-client-leave") { + return Scenario::RoomClientLeave; + } + if (value == "room-server-delete") { + return Scenario::RoomServerDelete; + } + throw std::runtime_error("invalid value for --scenario"); +} + +std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { + std::uint64_t parsed = 0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { + throw std::runtime_error("invalid value for " + std::string(option)); + } + return parsed; +} + +Options parseOptions(int argc, char* argv[]) { + Options options; + for (int index = 1; index < argc; ++index) { + const std::string_view option(argv[index]); + if (index + 1 >= argc) { + throw std::runtime_error("missing value for " + std::string(option)); + } + const std::string_view value(argv[++index]); + if (option == "--scenario") { + options.scenario = parseScenario(value); + } else if (option == "--iterations") { + options.iterations = parseUnsigned(value, option); + } else if (option == "--warmup") { + options.warmup = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-kib") { + options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-thread-growth") { + options.max_thread_growth = parseUnsigned(value, option); + } else { + throw std::runtime_error("unknown option: " + std::string(option)); + } + } + if (options.warmup == 0 || options.warmup >= options.iterations) { + throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); + } + return options; +} + +std::uint64_t statusValue(std::string_view line) { + const auto separator = line.find(':'); + if (separator == std::string_view::npos) { + throw std::runtime_error("malformed /proc/self/status value"); + } + line.remove_prefix(separator + 1); + while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { + line.remove_prefix(1); + } + const auto end = line.find_first_not_of("0123456789"); + return parseUnsigned(line.substr(0, end), "/proc/self/status"); +} + +ProcessSample sampleProcess() { + ProcessSample sample; + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmRSS:", 0) == 0) { + sample.rss_kib = statusValue(line); + } else if (line.rfind("Threads:", 0) == 0) { + sample.threads = statusValue(line); + } + } + if (sample.rss_kib == 0 || sample.threads == 0) { + throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); + } + return sample; +} + +RoomCredentials roomCredentials() { + const char* url = std::getenv("LIVEKIT_URL"); + const char* token = std::getenv("LIVEKIT_TOKEN_A"); + if (url == nullptr || token == nullptr || *url == '\0' || *token == '\0') { + throw std::runtime_error("room scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); + } + return {url, token}; +} + +void runSdkSourcesCycle() { + SdkLifetime sdk; + auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!audio_track || !video_track) { + throw std::runtime_error("failed to create local tracks"); + } +} + +void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { + if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { + throw std::runtime_error("room connection failed"); + } + if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { + throw std::runtime_error("room did not reach a connected state"); + } +} + +void runRoomClientLeaveCycle(const RoomCredentials& credentials) { + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("client room disconnect did not tear down local state"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); +} + +void runRoomServerDeleteCycle(const RoomCredentials& credentials) { + if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { + throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); + } + + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + // The room name is fixed rather than interpolating untrusted environment data into the command. + if (std::system("lk --dev room delete --yes cpp_data_track_test") != 0) { + throw std::runtime_error("failed to delete local test room"); + } + if (!delegate.waitForTeardown(10s)) { + throw std::runtime_error("timed out waiting for server room teardown"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("server room deletion did not tear down local state"); + } + delegate.requireServerTeardown(); +} + +void runCycle(Scenario scenario, const RoomCredentials& credentials) { + switch (scenario) { + case Scenario::SdkSources: + runSdkSourcesCycle(); + return; + case Scenario::RoomClientLeave: + runRoomClientLeaveCycle(credentials); + return; + case Scenario::RoomServerDelete: + runRoomServerDeleteCycle(credentials); + return; + } + throw std::runtime_error("unknown scenario"); +} + +} // namespace + +int main(int argc, char* argv[]) { + try { + const Options options = parseOptions(argc, argv); + const bool room_scenario = options.scenario != Scenario::SdkSources; + const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; + std::unique_ptr room_sdk; + if (room_scenario) { + room_sdk = std::make_unique(); + } + + ProcessSample baseline; + ProcessSample final; + for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { + runCycle(options.scenario, credentials); + std::this_thread::sleep_for(100ms); + if (iteration == options.warmup) { + baseline = sampleProcess(); + } else if (iteration == options.iterations) { + final = sampleProcess(); + } + } + + const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); + const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); + const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && + thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS " << baseline.rss_kib + << " -> " << final.rss_kib << " KiB (" << rss_growth << " KiB), threads " << baseline.threads << " -> " + << final.threads << " (" << thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/src/tests/memory/source_lifecycle_tester.cpp b/src/tests/memory/source_lifecycle_tester.cpp deleted file mode 100644 index 197da1ee..00000000 --- a/src/tests/memory/source_lifecycle_tester.cpp +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2026 LiveKit - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace { - -struct Options { - std::uint64_t iterations{100}; - std::uint64_t warmup{20}; - std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; - std::uint64_t max_thread_growth{0}; -}; - -struct ProcessSample { - std::uint64_t rss_kib{}; - std::uint64_t threads{}; -}; - -std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { - std::uint64_t parsed = 0; - const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); - if (result.ec != std::errc{} || result.ptr != value.data() + value.size()) { - throw std::runtime_error("invalid value for " + std::string(option)); - } - return parsed; -} - -Options parseOptions(int argc, char* argv[]) { - Options options; - for (int index = 1; index < argc; ++index) { - const std::string_view option(argv[index]); - if (index + 1 >= argc) { - throw std::runtime_error("missing value for " + std::string(option)); - } - const std::string_view value(argv[++index]); - if (option == "--iterations") { - options.iterations = parseUnsigned(value, option); - } else if (option == "--warmup") { - options.warmup = parseUnsigned(value, option); - } else if (option == "--max-rss-growth-kib") { - options.max_rss_growth_kib = parseUnsigned(value, option); - } else if (option == "--max-thread-growth") { - options.max_thread_growth = parseUnsigned(value, option); - } else { - throw std::runtime_error("unknown option: " + std::string(option)); - } - } - if (options.warmup == 0 || options.warmup >= options.iterations) { - throw std::runtime_error("--warmup must be between 1 and --iterations - 1"); - } - return options; -} - -std::uint64_t statusValue(std::string_view line) { - const auto separator = line.find(':'); - if (separator == std::string_view::npos) { - throw std::runtime_error("malformed /proc/self/status value"); - } - line.remove_prefix(separator + 1); - while (!line.empty() && std::isspace(static_cast(line.front())) != 0) { - line.remove_prefix(1); - } - const auto end = line.find_first_not_of("0123456789"); - return parseUnsigned(line.substr(0, end), "/proc/self/status"); -} - -ProcessSample sampleProcess() { - ProcessSample sample; - std::ifstream status("/proc/self/status"); - std::string line; - while (std::getline(status, line)) { - if (line.rfind("VmRSS:", 0) == 0) { - sample.rss_kib = statusValue(line); - } else if (line.rfind("Threads:", 0) == 0) { - sample.threads = statusValue(line); - } - } - if (sample.rss_kib == 0 || sample.threads == 0) { - throw std::runtime_error("failed to read RSS or thread count from /proc/self/status"); - } - return sample; -} - -void runCycle() { - if (!livekit::initialize(livekit::LogLevel::Warn)) { - throw std::runtime_error("LiveKit initialization failed"); - } - try { - { - auto audio_source = std::make_shared(48'000, 1, 100); - auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); - auto video_source = std::make_shared(640, 360); - auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); - if (!audio_track || !video_track) { - throw std::runtime_error("failed to create local tracks"); - } - } - } catch (...) { - livekit::shutdown(); - throw; - } - livekit::shutdown(); - std::this_thread::sleep_for(std::chrono::milliseconds(100)); -} - -} // namespace - -int main(int argc, char* argv[]) { - try { - const Options options = parseOptions(argc, argv); - ProcessSample baseline; - ProcessSample final; - for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { - runCycle(); - if (iteration == options.warmup) { - baseline = sampleProcess(); - } else if (iteration == options.iterations) { - final = sampleProcess(); - } - } - - const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); - const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); - const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && - thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: RSS " << baseline.rss_kib << " -> " << final.rss_kib << " KiB (" << rss_growth - << " KiB), threads " << baseline.threads << " -> " << final.threads << " (" << thread_growth - << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; - return passed ? 0 : 1; - } catch (const std::exception& error) { - std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; - return 1; - } -} From a92fea49f7a9b71f663d56b4bcaa72ce720ae751 Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:31:38 -0700 Subject: [PATCH 04/17] Split audio/video track --- docs/testing.md | 18 +++++---- scripts/memory-regression.sh | 12 +++--- src/tests/memory/lifecycle_tester.cpp | 57 +++++++++++++++++++++------ 3 files changed, 61 insertions(+), 26 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 90ebb4d5..3e7c264d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,10 +109,12 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate runs three lifecycle scenarios in separate processes: +The Linux PR gate runs four lifecycle scenarios in separate processes: -- `sdk-sources` repeatedly initializes the SDK, creates and destroys unused - audio/video sources and tracks, then shuts the SDK down. +- `audio-track` repeatedly initializes the SDK, creates an audio source and + track, captures synthetic PCM frames, destroys them, and shuts the SDK down. +- `video-track` repeatedly initializes the SDK, creates a video source and + track, captures synthetic I420 frames, destroys them, and shuts the SDK down. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. @@ -125,18 +127,20 @@ glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -The offline source scenario requires no LiveKit server or tokens: +The offline track scenarios require no LiveKit server or tokens: ```bash ./build.sh release-tests -scripts/memory-regression.sh \ - --scenario sdk-sources \ +scripts/memory-regression.sh --scenario audio-track \ + --iterations 100 --warmup 20 \ + --max-rss-growth-kib 8192 --max-thread-growth 0 +scripts/memory-regression.sh --scenario video-track \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` To run either room scenario, start `livekit-server --dev`, install the `lk` -CLI, and source the local test tokens. Use `--scenario all` to run all three +CLI, and source the local test tokens. Use `--scenario all` to run all four scenarios, each in a fresh tester process: ```bash diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 1032d769..74ba29da 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,8 +25,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) - --scenario NAME sdk-sources, room-client-leave, - room-server-delete, or all (default: sdk-sources) + --scenario NAME audio-track, video-track, room-client-leave, + room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -44,7 +44,7 @@ EOF repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" -scenario="${MEMORY_REGRESSION_SCENARIO:-sdk-sources}" +scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" @@ -73,11 +73,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - sdk-sources|room-client-leave|room-server-delete) + audio-track|video-track|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(sdk-sources room-client-leave room-server-delete) + scenarios=(audio-track video-track room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -87,7 +87,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" != "sdk-sources" ]] && + if [[ "${current_scenario}" == room-* ]] && [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 8fbb7535..46576c9d 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -16,6 +16,7 @@ #include +#include #include #include #include @@ -36,15 +37,17 @@ namespace { using namespace std::chrono_literals; constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; +constexpr int kFramesPerCycle = 3; enum class Scenario { - SdkSources, + AudioTrack, + VideoTrack, RoomClientLeave, RoomServerDelete, }; struct Options { - Scenario scenario{Scenario::SdkSources}; + Scenario scenario{Scenario::AudioTrack}; std::uint64_t iterations{100}; std::uint64_t warmup{20}; std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; @@ -123,8 +126,10 @@ class DisconnectTrackingDelegate final : public livekit::RoomDelegate { std::string_view scenarioName(Scenario scenario) { switch (scenario) { - case Scenario::SdkSources: - return "sdk-sources"; + case Scenario::AudioTrack: + return "audio-track"; + case Scenario::VideoTrack: + return "video-track"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -134,8 +139,11 @@ std::string_view scenarioName(Scenario scenario) { } Scenario parseScenario(std::string_view value) { - if (value == "sdk-sources") { - return Scenario::SdkSources; + if (value == "audio-track") { + return Scenario::AudioTrack; + } + if (value == "video-track") { + return Scenario::VideoTrack; } if (value == "room-client-leave") { return Scenario::RoomClientLeave; @@ -222,14 +230,33 @@ RoomCredentials roomCredentials() { return {url, token}; } -void runSdkSourcesCycle() { +void runAudioTrackCycle() { SdkLifetime sdk; - auto audio_source = std::make_shared(48'000, 1, 100); + auto audio_source = std::make_shared(48'000, 1); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); + if (!audio_track) { + throw std::runtime_error("failed to create local audio track"); + } + + auto frame = livekit::AudioFrame::create(48'000, 1, 480); + std::fill(frame.data().begin(), frame.data().end(), 100); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + audio_source->captureFrame(frame, 1'000); + } +} + +void runVideoTrackCycle() { + SdkLifetime sdk; auto video_source = std::make_shared(640, 360); auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); - if (!audio_track || !video_track) { - throw std::runtime_error("failed to create local tracks"); + if (!video_track) { + throw std::runtime_error("failed to create local video track"); + } + + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); + std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + video_source->captureFrame(frame); } } @@ -282,8 +309,11 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { void runCycle(Scenario scenario, const RoomCredentials& credentials) { switch (scenario) { - case Scenario::SdkSources: - runSdkSourcesCycle(); + case Scenario::AudioTrack: + runAudioTrackCycle(); + return; + case Scenario::VideoTrack: + runVideoTrackCycle(); return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); @@ -300,7 +330,8 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool room_scenario = options.scenario != Scenario::SdkSources; + const bool room_scenario = + options.scenario == Scenario::RoomClientLeave || options.scenario == Scenario::RoomServerDelete; const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; std::unique_ptr room_sdk; if (room_scenario) { From 8ba2935cb17c70413583225ac49f356dc491150a Mon Sep 17 00:00:00 2001 From: Alan George Date: Wed, 7 Oct 2026 20:46:37 -0700 Subject: [PATCH 05/17] Warm up stats vs. final --- docs/testing.md | 15 +++-- scripts/memory-regression.sh | 25 +++++---- src/tests/memory/lifecycle_tester.cpp | 80 ++++++++++++++++++++++----- 3 files changed, 91 insertions(+), 29 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 3e7c264d..2378b9bf 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -109,18 +109,22 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - ## Linux memory regression gate -The Linux PR gate runs four lifecycle scenarios in separate processes: +The Linux PR gate runs five lifecycle scenarios in separate processes: - `audio-track` repeatedly initializes the SDK, creates an audio source and track, captures synthetic PCM frames, destroys them, and shuts the SDK down. - `video-track` repeatedly initializes the SDK, creates a video source and track, captures synthetic I420 frames, destroys them, and shuts the SDK down. +- `data-track` repeatedly connects, publishes a data track, pushes synthetic + payloads, unpublishes it, and disconnects. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. -Each scenario compares process RSS and thread count after cycle 20 with cycle -100, failing on thread growth or more than 8 MiB of RSS growth. +Each scenario reports process RSS and thread count at cycle 0, after warmup +cycle 20, and after final cycle 100. The cycle 0 to warmup growth shows expected +one-time initialization; the gate applies to warmup-to-final growth and fails +on thread growth or more than 8 MiB of RSS growth. The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a @@ -139,12 +143,13 @@ scripts/memory-regression.sh --scenario video-track \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` -To run either room scenario, start `livekit-server --dev`, install the `lk` -CLI, and source the local test tokens. Use `--scenario all` to run all four +To run the data-track or room scenarios, start `livekit-server --dev`, install +the `lk` CLI, and source the local test tokens. Use `--scenario all` to run all five scenarios, each in a fresh tester process: ```bash source scripts/set-test-tokens.sh +scripts/memory-regression.sh --scenario data-track scripts/memory-regression.sh --scenario room-client-leave scripts/memory-regression.sh --scenario room-server-delete scripts/memory-regression.sh --scenario all diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 74ba29da..d805ffc2 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -25,8 +25,9 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) - --scenario NAME audio-track, video-track, room-client-leave, - room-server-delete, or all (default: audio-track) + --scenario NAME audio-track, video-track, data-track, + room-client-leave, room-server-delete, or all + (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -73,11 +74,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|room-client-leave|room-server-delete) + audio-track|video-track|data-track|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(audio-track video-track room-client-leave room-server-delete) + scenarios=(audio-track video-track data-track room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -87,7 +88,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" == room-* ]] && + if [[ "${current_scenario}" == "data-track" || "${current_scenario}" == room-* ]] && [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 @@ -99,14 +100,14 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { echo "## Linux memory regression" echo - echo "Limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." + echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | RSS baseline | RSS final | RSS growth | Threads baseline | Threads final | Thread growth | Verdict |" - echo "|---|---:|---:|---:|---:|---:|---:|---|" + echo "| Scenario | Metric | Cycle 0 | Warmup (${warmup}) | Final (${iterations}) | 0 → warmup | Warmup → final | Verdict |" + echo "|---|---|---:|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS ([0-9]+) -> ([0-9]+) KiB \((-?[0-9]+) KiB\), threads ([0-9]+) -> ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' overall_status=0 for current_scenario in "${scenarios[@]}"; do set +e @@ -124,8 +125,10 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${BASH_REMATCH[1]} | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[5]} | ${BASH_REMATCH[6]} | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} |" \ - >> "${GITHUB_STEP_SUMMARY}" + { + echo "| ${BASH_REMATCH[1]} | RSS | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[5]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[6]} KiB | ${BASH_REMATCH[12]} |" + echo "| ${BASH_REMATCH[1]} | Threads | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} | ${BASH_REMATCH[10]} | ${BASH_REMATCH[9]} | ${BASH_REMATCH[11]} | ${BASH_REMATCH[12]} |" + } >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 46576c9d..7c8cfc9d 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -42,6 +43,7 @@ constexpr int kFramesPerCycle = 3; enum class Scenario { AudioTrack, VideoTrack, + DataTrack, RoomClientLeave, RoomServerDelete, }; @@ -130,6 +132,8 @@ std::string_view scenarioName(Scenario scenario) { return "audio-track"; case Scenario::VideoTrack: return "video-track"; + case Scenario::DataTrack: + return "data-track"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -145,6 +149,9 @@ Scenario parseScenario(std::string_view value) { if (value == "video-track") { return Scenario::VideoTrack; } + if (value == "data-track") { + return Scenario::DataTrack; + } if (value == "room-client-leave") { return Scenario::RoomClientLeave; } @@ -269,6 +276,41 @@ void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { } } +void runDataTrackCycle(const RoomCredentials& credentials) { + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing data track"); + } + auto publish_result = participant->publishDataTrack("memory-data"); + if (!publish_result) { + throw std::runtime_error("failed to publish local data track"); + } + + const auto& track = publish_result.value(); + livekit::DataTrackFrame frame; + frame.payload.assign(256, 0xFA); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + frame.user_timestamp = static_cast(frame_index); + if (!track->tryPush(frame)) { + throw std::runtime_error("failed to push local data track frame"); + } + } + + track->unpublishDataTrack(); + if (track->isPublished()) { + throw std::runtime_error("local data track remained published"); + } + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); +} + void runRoomClientLeaveCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -315,6 +357,9 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { case Scenario::VideoTrack: runVideoTrackCycle(); return; + case Scenario::DataTrack: + runDataTrackCycle(credentials); + return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); return; @@ -330,33 +375,42 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool room_scenario = - options.scenario == Scenario::RoomClientLeave || options.scenario == Scenario::RoomServerDelete; - const RoomCredentials credentials = room_scenario ? roomCredentials() : RoomCredentials{}; + const bool connected_scenario = options.scenario == Scenario::DataTrack || + options.scenario == Scenario::RoomClientLeave || + options.scenario == Scenario::RoomServerDelete; + const RoomCredentials credentials = connected_scenario ? roomCredentials() : RoomCredentials{}; std::unique_ptr room_sdk; - if (room_scenario) { + if (connected_scenario) { room_sdk = std::make_unique(); } - ProcessSample baseline; + const ProcessSample initial = sampleProcess(); + ProcessSample warmup; ProcessSample final; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { runCycle(options.scenario, credentials); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { - baseline = sampleProcess(); + warmup = sampleProcess(); } else if (iteration == options.iterations) { final = sampleProcess(); } } - const auto rss_growth = static_cast(final.rss_kib) - static_cast(baseline.rss_kib); - const auto thread_growth = static_cast(final.threads) - static_cast(baseline.threads); - const bool passed = rss_growth <= static_cast(options.max_rss_growth_kib) && - thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS " << baseline.rss_kib - << " -> " << final.rss_kib << " KiB (" << rss_growth << " KiB), threads " << baseline.threads << " -> " - << final.threads << " (" << thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + const auto warmup_rss_growth = + static_cast(warmup.rss_kib) - static_cast(initial.rss_kib); + const auto final_rss_growth = static_cast(final.rss_kib) - static_cast(warmup.rss_kib); + const auto warmup_thread_growth = + static_cast(warmup.threads) - static_cast(initial.threads); + const auto final_thread_growth = + static_cast(final.threads) - static_cast(warmup.threads); + const bool passed = final_rss_growth <= static_cast(options.max_rss_growth_kib) && + final_thread_growth <= static_cast(options.max_thread_growth); + std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << initial.rss_kib + << " -> warmup " << warmup.rss_kib << " KiB (" << warmup_rss_growth << " KiB) -> final " << final.rss_kib + << " KiB (" << final_rss_growth << " KiB), threads 0 " << initial.threads << " -> warmup " + << warmup.threads << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" + << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; From 682f6d6e680fdf4f4fc2eed86470f65f4703882b Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 8 Oct 2026 09:39:58 -0700 Subject: [PATCH 06/17] Cleanup docs, comments, add room reuse --- .github/workflows/tests.yml | 2 - docs/testing.md | 44 +++++----- scripts/memory-regression.sh | 38 +++++--- src/tests/memory/lifecycle_tester.cpp | 121 ++++++++++++++++++++++---- 4 files changed, 150 insertions(+), 55 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9accd92d..187dacbd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -437,8 +437,6 @@ jobs: if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash - env: - MALLOC_ARENA_MAX: "1" run: | set -euo pipefail source scripts/set-test-tokens.sh diff --git a/docs/testing.md b/docs/testing.md index 2378b9bf..2e5f64b4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -107,34 +107,42 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. -## Linux memory regression gate +## Memory lifecycle tester -The Linux PR gate runs five lifecycle scenarios in separate processes: +> Note: Linux only -- `audio-track` repeatedly initializes the SDK, creates an audio source and - track, captures synthetic PCM frames, destroys them, and shuts the SDK down. -- `video-track` repeatedly initializes the SDK, creates a video source and - track, captures synthetic I420 frames, destroys them, and shuts the SDK down. +The memory lifecycle tester binary catches resource leak regressions in an +automated fashion, and is invoked in CI. This single binary runs individual +or groups of scenarios: + +- `audio-track` repeatedly initializes the SDK, connects, publishes an audio + track, captures synthetic PCM frames, unpublishes, disconnects, and shuts down. +- `video-track` repeatedly initializes the SDK, connects, publishes a video + track, captures synthetic I420 frames, unpublishes, disconnects, and shuts down. - `data-track` repeatedly connects, publishes a data track, pushes synthetic payloads, unpublishes it, and disconnects. +- `room-reuse` repeatedly connects and disconnects the same room object. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. - `room-server-delete` repeatedly connects and uses `lk` to delete the local test room, then verifies the disconnect and end-of-stream callbacks. Each scenario reports process RSS and thread count at cycle 0, after warmup -cycle 20, and after final cycle 100. The cycle 0 to warmup growth shows expected -one-time initialization; the gate applies to warmup-to-final growth and fails -on thread growth or more than 8 MiB of RSS growth. +cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth +and fails on thread growth or more than the configured amount of RSS growth. -The workflow sets `MALLOC_ARENA_MAX=1` before process launch. This limits -glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a +The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, +which invokes the binary but also has GitHub actions support for a results summary table. +The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. +This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a controlled CI signal, not evidence that the default allocator returns all freed pages to the OS. -The offline track scenarios require no LiveKit server or tokens: +Start `livekit-server --dev`, install the `lk` CLI, and source the local test +tokens before running any scenario: ```bash ./build.sh release-tests +source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario audio-track \ --iterations 100 --warmup 20 \ --max-rss-growth-kib 8192 --max-thread-growth 0 @@ -143,24 +151,20 @@ scripts/memory-regression.sh --scenario video-track \ --max-rss-growth-kib 8192 --max-thread-growth 0 ``` -To run the data-track or room scenarios, start `livekit-server --dev`, install -the `lk` CLI, and source the local test tokens. Use `--scenario all` to run all five -scenarios, each in a fresh tester process: +Use `--scenario all` to run all six scenarios, each in a fresh tester process: ```bash -source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario data-track +scripts/memory-regression.sh --scenario room-reuse scripts/memory-regression.sh --scenario room-client-leave scripts/memory-regression.sh --scenario room-server-delete scripts/memory-regression.sh --scenario all ``` The limits are command-line options so a longer local run can use different -budgets. The wrapper defaults `MALLOC_ARENA_MAX` to `1` and appends one results -row per scenario when `GITHUB_STEP_SUMMARY` is set. Hardware encoding coverage -is separate follow-up work. +budgets. -## Memory checks (valgrind) +## Additional memory checks (valgrind) Run `valgrind` against the test binaries to check for memory leaks and other issues. See [tools.md](tools.md) for the recipe. diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index d805ffc2..056baa79 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -26,8 +26,8 @@ GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: --build-dir PATH Build directory (default: build-release) --scenario NAME audio-track, video-track, data-track, - room-client-leave, room-server-delete, or all - (default: audio-track) + room-reuse, room-client-leave, + room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-kib N Allowed RSS growth (default: 8192) @@ -39,7 +39,7 @@ Environment: MEMORY_REGRESSION_ITERATIONS, MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. - Room scenarios also require LIVEKIT_URL and LIVEKIT_TOKEN_A. + All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } @@ -74,11 +74,11 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|data-track|room-client-leave|room-server-delete) + audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete) scenarios=("${scenario}") ;; all) - scenarios=(audio-track video-track data-track room-client-leave room-server-delete) + scenarios=(audio-track video-track data-track room-reuse room-client-leave room-server-delete) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 @@ -88,8 +88,7 @@ case "${scenario}" in esac for current_scenario in "${scenarios[@]}"; do - if [[ "${current_scenario}" == "data-track" || "${current_scenario}" == room-* ]] && - [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then + if [[ -z "${LIVEKIT_URL:-}" || -z "${LIVEKIT_TOKEN_A:-}" ]]; then echo "ERROR: ${current_scenario} requires LIVEKIT_URL and LIVEKIT_TOKEN_A." >&2 echo "Run: source scripts/set-test-tokens.sh" >&2 exit 1 @@ -102,12 +101,21 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then echo echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | Metric | Cycle 0 | Warmup (${warmup}) | Final (${iterations}) | 0 → warmup | Warmup → final | Verdict |" - echo "|---|---|---:|---:|---:|---:|---:|---|" + echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} |" + echo "|---|---:|---:|---:|---:|" } >> "${GITHUB_STEP_SUMMARY}" fi report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' + +format_growth() { + if [[ "$1" == "0" || "$1" == -* ]]; then + printf '%s' "$1" + else + printf '+%s' "$1" + fi +} + overall_status=0 for current_scenario in "${scenarios[@]}"; do set +e @@ -125,15 +133,17 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - { - echo "| ${BASH_REMATCH[1]} | RSS | ${BASH_REMATCH[2]} KiB | ${BASH_REMATCH[3]} KiB | ${BASH_REMATCH[5]} KiB | ${BASH_REMATCH[4]} KiB | ${BASH_REMATCH[6]} KiB | ${BASH_REMATCH[12]} |" - echo "| ${BASH_REMATCH[1]} | Threads | ${BASH_REMATCH[7]} | ${BASH_REMATCH[8]} | ${BASH_REMATCH[10]} | ${BASH_REMATCH[9]} | ${BASH_REMATCH[11]} | ${BASH_REMATCH[12]} |" - } >> "${GITHUB_STEP_SUMMARY}" + rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")" + rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")" + thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")" + thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")" + echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} KiB | ${rss_final_growth} KiB | ${thread_warmup_growth} | ${thread_final_growth} |" \ + >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${current_scenario} | — | — | — | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + echo "| ${current_scenario} | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" fi status=1 fi diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index 7c8cfc9d..fb9f0462 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -44,6 +44,7 @@ enum class Scenario { AudioTrack, VideoTrack, DataTrack, + RoomReuse, RoomClientLeave, RoomServerDelete, }; @@ -134,6 +135,8 @@ std::string_view scenarioName(Scenario scenario) { return "video-track"; case Scenario::DataTrack: return "data-track"; + case Scenario::RoomReuse: + return "room-reuse"; case Scenario::RoomClientLeave: return "room-client-leave"; case Scenario::RoomServerDelete: @@ -152,6 +155,9 @@ Scenario parseScenario(std::string_view value) { if (value == "data-track") { return Scenario::DataTrack; } + if (value == "room-reuse") { + return Scenario::RoomReuse; + } if (value == "room-client-leave") { return Scenario::RoomClientLeave; } @@ -232,50 +238,103 @@ RoomCredentials roomCredentials() { const char* url = std::getenv("LIVEKIT_URL"); const char* token = std::getenv("LIVEKIT_TOKEN_A"); if (url == nullptr || token == nullptr || *url == '\0' || *token == '\0') { - throw std::runtime_error("room scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); + throw std::runtime_error("memory scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A"); } return {url, token}; } -void runAudioTrackCycle() { +void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { + if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { + throw std::runtime_error("room connection failed"); + } + if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { + throw std::runtime_error("room did not reach a connected state"); + } +} + +// Exercises SDK startup, room connection, audio publication and capture, +// explicit unpublication, client disconnect, and complete process-side teardown. +void runAudioTrackCycle(const RoomCredentials& credentials) { SdkLifetime sdk; + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + auto audio_source = std::make_shared(48'000, 1); auto audio_track = livekit::LocalAudioTrack::createLocalAudioTrack("memory-audio", audio_source); if (!audio_track) { throw std::runtime_error("failed to create local audio track"); } + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing audio track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_MICROPHONE; + participant->publishTrack(audio_track, publish_options); + const auto publication = audio_track->publication(); + if (!publication) { + throw std::runtime_error("failed to publish local audio track"); + } + auto frame = livekit::AudioFrame::create(48'000, 1, 480); std::fill(frame.data().begin(), frame.data().end(), 100); for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { audio_source->captureFrame(frame, 1'000); } + + participant->unpublishTrack(publication->sid()); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } -void runVideoTrackCycle() { +// Exercises SDK startup, room connection, video publication and capture, +// encoder/resource teardown, explicit unpublication, and client disconnect. +void runVideoTrackCycle(const RoomCredentials& credentials) { SdkLifetime sdk; + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + auto video_source = std::make_shared(640, 360); auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); if (!video_track) { throw std::runtime_error("failed to create local video track"); } + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing video track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + participant->publishTrack(video_track, publish_options); + const auto publication = video_track->publication(); + if (!publication) { + throw std::runtime_error("failed to publish local video track"); + } + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { video_source->captureFrame(frame); } -} -void connectRoom(livekit::Room& room, const RoomCredentials& credentials) { - if (!room.connect(credentials.url, credentials.token, livekit::RoomOptions{})) { - throw std::runtime_error("room connection failed"); - } - if (room.connectionState() != livekit::ConnectionState::Connected || room.localParticipant().expired()) { - throw std::runtime_error("room did not reach a connected state"); + participant->unpublishTrack(publication->sid()); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises data-track publication, payload enqueueing, explicit unpublication, +// and room teardown while the SDK remains initialized across cycles. void runDataTrackCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -311,6 +370,20 @@ void runDataTrackCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises repeated connect/disconnect cycles on one Room instance to detect +// connection state, participant, listener, or FFI resources retained by reuse. +void runRoomReuseCycle(livekit::Room& room, const RoomCredentials& credentials) { + connectRoom(room, credentials); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + if (room.connectionState() != livekit::ConnectionState::Disconnected || !room.localParticipant().expired()) { + throw std::runtime_error("reused room disconnect did not tear down local state"); + } +} + +// Exercises application-initiated disconnect and destruction of a fresh Room, +// including local-participant cleanup and the expected disconnect callback. void runRoomClientLeaveCycle(const RoomCredentials& credentials) { DisconnectTrackingDelegate delegate; livekit::Room room; @@ -326,6 +399,8 @@ void runRoomClientLeaveCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } +// Exercises server-initiated room deletion and validates disconnect, event-stream +// completion, local-participant cleanup, and destruction of the torn-down Room. void runRoomServerDeleteCycle(const RoomCredentials& credentials) { if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); @@ -349,17 +424,23 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { delegate.requireServerTeardown(); } -void runCycle(Scenario scenario, const RoomCredentials& credentials) { +void runCycle(Scenario scenario, const RoomCredentials& credentials, livekit::Room* reusable_room) { switch (scenario) { case Scenario::AudioTrack: - runAudioTrackCycle(); + runAudioTrackCycle(credentials); return; case Scenario::VideoTrack: - runVideoTrackCycle(); + runVideoTrackCycle(credentials); return; case Scenario::DataTrack: runDataTrackCycle(credentials); return; + case Scenario::RoomReuse: + if (reusable_room == nullptr) { + throw std::runtime_error("room-reuse scenario is missing its persistent room"); + } + runRoomReuseCycle(*reusable_room, credentials); + return; case Scenario::RoomClientLeave: runRoomClientLeaveCycle(credentials); return; @@ -375,20 +456,22 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials) { int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); - const bool connected_scenario = options.scenario == Scenario::DataTrack || - options.scenario == Scenario::RoomClientLeave || - options.scenario == Scenario::RoomServerDelete; - const RoomCredentials credentials = connected_scenario ? roomCredentials() : RoomCredentials{}; + const RoomCredentials credentials = roomCredentials(); + const bool per_cycle_sdk = options.scenario == Scenario::AudioTrack || options.scenario == Scenario::VideoTrack; std::unique_ptr room_sdk; - if (connected_scenario) { + if (!per_cycle_sdk) { room_sdk = std::make_unique(); } + std::unique_ptr reusable_room; + if (options.scenario == Scenario::RoomReuse) { + reusable_room = std::make_unique(); + } const ProcessSample initial = sampleProcess(); ProcessSample warmup; ProcessSample final; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { - runCycle(options.scenario, credentials); + runCycle(options.scenario, credentials, reusable_room.get()); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { warmup = sampleProcess(); From d638cca74f5fc1b4b53ba1c323416d0e4b0ee8c0 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 8 Oct 2026 11:33:09 -0700 Subject: [PATCH 07/17] Megabytes, emojis --- .github/workflows/tests.yml | 4 ++-- docs/testing.md | 4 ++-- scripts/memory-regression.sh | 33 ++++++++++++++++----------- src/tests/memory/lifecycle_tester.cpp | 24 +++++++++++-------- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 187dacbd..4d87c24f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -433,7 +433,7 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml - - name: Run Linux memory regression gate + - name: Run memory regression tests if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash @@ -445,7 +445,7 @@ jobs: --scenario all \ --iterations 100 \ --warmup 20 \ - --max-rss-growth-kib 8192 \ + --max-rss-growth-mib 8 \ --max-thread-growth 0 - name: Run stress tests diff --git a/docs/testing.md b/docs/testing.md index 2e5f64b4..20b7c673 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -145,10 +145,10 @@ tokens before running any scenario: source scripts/set-test-tokens.sh scripts/memory-regression.sh --scenario audio-track \ --iterations 100 --warmup 20 \ - --max-rss-growth-kib 8192 --max-thread-growth 0 + --max-rss-growth-mib 8 --max-thread-growth 0 scripts/memory-regression.sh --scenario video-track \ --iterations 100 --warmup 20 \ - --max-rss-growth-kib 8192 --max-thread-growth 0 + --max-rss-growth-mib 8 --max-thread-growth 0 ``` Use `--scenario all` to run all six scenarios, each in a fresh tester process: diff --git a/scripts/memory-regression.sh b/scripts/memory-regression.sh index 056baa79..cfee8beb 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-regression.sh @@ -30,14 +30,14 @@ Options: room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) - --max-rss-growth-kib N Allowed RSS growth (default: 8192) + --max-rss-growth-mib N Allowed RSS growth (default: 8) --max-thread-growth N Allowed thread growth (default: 0) -h, --help Show this help Environment: MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, MEMORY_REGRESSION_ITERATIONS, - MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB, + MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB, MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF @@ -48,7 +48,7 @@ build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" warmup="${MEMORY_REGRESSION_WARMUP:-20}" -max_rss_growth_kib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_KIB:-8192}" +max_rss_growth_mib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB:-8}" max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" while (($#)); do @@ -57,7 +57,7 @@ while (($#)); do --scenario) scenario="$2"; shift 2 ;; --iterations) iterations="$2"; shift 2 ;; --warmup) warmup="$2"; shift 2 ;; - --max-rss-growth-kib) max_rss_growth_kib="$2"; shift 2 ;; + --max-rss-growth-mib) max_rss_growth_mib="$2"; shift 2 ;; --max-thread-growth) max_thread_growth="$2"; shift 2 ;; -h|--help) usage; exit 0 ;; *) echo "ERROR: unknown option: $1" >&2; usage >&2; exit 2 ;; @@ -97,19 +97,21 @@ done if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { - echo "## Linux memory regression" + echo "## Memory regression results" echo - echo "Warmup → final limits: RSS growth ≤ ${max_rss_growth_kib} KiB; thread growth ≤ ${max_thread_growth}." + echo "Iterations: ${iterations}" + echo "Warmup: ${warmup}" + echo "Threshold: RSS growth ≤ ${max_rss_growth_mib} MiB; thread growth ≤ ${max_thread_growth}." echo - echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} |" - echo "|---|---:|---:|---:|---:|" + echo "| Scenario | RSS 0→${warmup} | RSS ${warmup}→${iterations} | Threads 0→${warmup} | Threads ${warmup}→${iterations} | Status |" + echo "|---|---:|---:|---:|---:|---|" } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+) -> warmup ([0-9]+) KiB \((-?[0-9]+) KiB\) -> final ([0-9]+) KiB \((-?[0-9]+) KiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' format_growth() { - if [[ "$1" == "0" || "$1" == -* ]]; then + if [[ "$1" == "0" || "$1" == "0.00" || "$1" == -* ]]; then printf '%s' "$1" else printf '+%s' "$1" @@ -124,7 +126,7 @@ for current_scenario in "${scenarios[@]}"; do --scenario "${current_scenario}" \ --iterations "${iterations}" \ --warmup "${warmup}" \ - --max-rss-growth-kib "${max_rss_growth_kib}" \ + --max-rss-growth-mib "${max_rss_growth_mib}" \ --max-thread-growth "${max_thread_growth}" 2>&1 )" status=$? @@ -133,17 +135,22 @@ for current_scenario in "${scenarios[@]}"; do if [[ "${output}" =~ ${report_regex} ]]; then if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + if [[ "${BASH_REMATCH[12]}" == "PASS" ]]; then + verdict=":white_check_mark: PASS" + else + verdict=":x: FAIL" + fi rss_warmup_growth="$(format_growth "${BASH_REMATCH[4]}")" rss_final_growth="$(format_growth "${BASH_REMATCH[6]}")" thread_warmup_growth="$(format_growth "${BASH_REMATCH[9]}")" thread_final_growth="$(format_growth "${BASH_REMATCH[11]}")" - echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} KiB | ${rss_final_growth} KiB | ${thread_warmup_growth} | ${thread_final_growth} |" \ + echo "| ${BASH_REMATCH[1]} | ${rss_warmup_growth} MiB | ${rss_final_growth} MiB | ${thread_warmup_growth} | ${thread_final_growth} | ${verdict} |" \ >> "${GITHUB_STEP_SUMMARY}" fi else echo "ERROR: ${current_scenario} did not produce a memory report." >&2 if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then - echo "| ${current_scenario} | — | — | — | ERROR |" >> "${GITHUB_STEP_SUMMARY}" + echo "| ${current_scenario} | — | — | — | — | :x: ERROR |" >> "${GITHUB_STEP_SUMMARY}" fi status=1 fi diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/lifecycle_tester.cpp index fb9f0462..b8d279ad 100644 --- a/src/tests/memory/lifecycle_tester.cpp +++ b/src/tests/memory/lifecycle_tester.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -53,7 +54,7 @@ struct Options { Scenario scenario{Scenario::AudioTrack}; std::uint64_t iterations{100}; std::uint64_t warmup{20}; - std::uint64_t max_rss_growth_kib{std::uint64_t{8} * 1024U}; + std::uint64_t max_rss_growth_mib{8}; std::uint64_t max_thread_growth{0}; }; @@ -190,8 +191,8 @@ Options parseOptions(int argc, char* argv[]) { options.iterations = parseUnsigned(value, option); } else if (option == "--warmup") { options.warmup = parseUnsigned(value, option); - } else if (option == "--max-rss-growth-kib") { - options.max_rss_growth_kib = parseUnsigned(value, option); + } else if (option == "--max-rss-growth-mib") { + options.max_rss_growth_mib = parseUnsigned(value, option); } else if (option == "--max-thread-growth") { options.max_thread_growth = parseUnsigned(value, option); } else { @@ -234,6 +235,10 @@ ProcessSample sampleProcess() { return sample; } +double kibToMib(std::uint64_t kib) { return static_cast(kib) / 1024.0; } + +double kibToMib(std::int64_t kib) { return static_cast(kib) / 1024.0; } + RoomCredentials roomCredentials() { const char* url = std::getenv("LIVEKIT_URL"); const char* token = std::getenv("LIVEKIT_TOKEN_A"); @@ -487,13 +492,14 @@ int main(int argc, char* argv[]) { static_cast(warmup.threads) - static_cast(initial.threads); const auto final_thread_growth = static_cast(final.threads) - static_cast(warmup.threads); - const bool passed = final_rss_growth <= static_cast(options.max_rss_growth_kib) && + const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); - std::cout << "memory lifecycle: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << initial.rss_kib - << " -> warmup " << warmup.rss_kib << " KiB (" << warmup_rss_growth << " KiB) -> final " << final.rss_kib - << " KiB (" << final_rss_growth << " KiB), threads 0 " << initial.threads << " -> warmup " - << warmup.threads << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" - << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + std::cout << std::fixed << std::setprecision(2) << "memory lifecycle: scenario=" << scenarioName(options.scenario) + << ", RSS 0 " << kibToMib(initial.rss_kib) << " -> warmup " << kibToMib(warmup.rss_kib) << " MiB (" + << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.rss_kib) << " MiB (" + << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.threads << " -> warmup " << warmup.threads + << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" << final_thread_growth + << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; From 9af5d015770febf5ccb76a94c8434e0fcb7b0dd3 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 13:23:38 -0600 Subject: [PATCH 08/17] Rename and update docs --- docs/testing.md | 59 ++++++++++++------- src/tests/CMakeLists.txt | 2 +- ...lifecycle_tester.cpp => memory_tester.cpp} | 0 3 files changed, 38 insertions(+), 23 deletions(-) rename src/tests/memory/{lifecycle_tester.cpp => memory_tester.cpp} (100%) diff --git a/docs/testing.md b/docs/testing.md index 20b7c673..cbf97cc5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1,7 +1,21 @@ # Testing -The SDK includes integration and stress tests using -[Google Test](https://github.com/google/googletest). +The SDK includes unit, integration and stress tests using +[Google Test](https://github.com/google/googletest). It also +includes a memory-leak/resource checking tester. + +Unit, integration, and memory tests are run on every PR in CI. +Stress tests are reserved for nightly tests in CI due to their +length. + +## Test binaries + +| Executable | Description | +|------------|-------------| +| `livekit_unit_tests` | Unit tests (no server required) | +| `livekit_integration_tests` | Integration tests (server required) | +| `livekit_stress_tests` | Long-running stress tests (server required) | +| `livekit_memory_lifecycle_tester` | Resource leak tester | ## Building the test binaries @@ -17,34 +31,35 @@ The SDK includes integration and stress tests using .\build.cmd release-tests ``` -## Running tests +After building, test binaries are available at: -After building, run tests using `ctest` or invoke the binaries directly: +```bash +# Linux/macOS +/bin/ + +# Windows +\bin\ +``` + +## Running tests ```bash -# Run all tests via ctest +# Run all tests via ctest (warning, runs unit, ) cd build-debug ctest --output-on-failure # Or run test executables directly +./build-debug/bin/livekit_unit_tests ./build-debug/bin/livekit_integration_tests ./build-debug/bin/livekit_stress_tests # Run specific test suites +./build-debug/bin/livekit_unit_tests --gtest_filter="*Room*" ./build-debug/bin/livekit_integration_tests --gtest_filter="*Rpc*" ./build-debug/bin/livekit_stress_tests --gtest_filter="*MaxPayloadStress*" ``` -__Note:__ The tests require tokens and a running LiveKit server. See the section below for details. - -## Test binaries - -| Executable | Description | -|------------|-------------| -| `livekit_unit_tests` | Pure unit tests (no server required) | -| `livekit_integration_tests` | Quick tests (~1-2 minutes) for SDK functionality | -| `livekit_stress_tests` | Long-running tests (configurable, default 1 hour) | -| `livekit_memory_lifecycle_tester` | Linux lifecycle RSS and thread regression scenarios | +__Note:__ The integration and stress tests require tokens and a running LiveKit server. See the section below for details. ## Running a local LiveKit server for tests @@ -53,9 +68,10 @@ path is `livekit-server --dev`, which uses the well-known dev API key/secret (`devkey` / `secret`). Install [`livekit-server`](https://docs.livekit.io/home/self-hosting/local/) -and start it with data tracks enabled: +and start it with data blobs enabled: ```bash +LIVEKIT_CONFIG="enable_participant_data_blob: true" livekit-server --dev ``` @@ -130,12 +146,11 @@ Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth and fails on thread growth or more than the configured amount of RSS growth. -The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, -which invokes the binary but also has GitHub actions support for a results summary table. -The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. -This limits glibc's allocator arenas, reducing Linux thread-churn RSS noise; it is a -controlled CI signal, not evidence that the default allocator returns all -freed pages to the OS. +The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, +which invokes the binary but also has GitHub actions support for a results summary table. +The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. +This limits `glibc`'s allocator arenas, reducing Linux thread-churn RSS noise, but it is +not evidence that the default allocator returns all freed pages to the OS. Start `livekit-server --dev`, install the `lk` CLI, and source the local test tokens before running any scenario: diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 1204193c..02e02bed 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -351,7 +351,7 @@ endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_executable(livekit_memory_lifecycle_tester - "${CMAKE_CURRENT_SOURCE_DIR}/memory/lifecycle_tester.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/memory/memory_tester.cpp" ) target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) target_include_directories(livekit_memory_lifecycle_tester PRIVATE diff --git a/src/tests/memory/lifecycle_tester.cpp b/src/tests/memory/memory_tester.cpp similarity index 100% rename from src/tests/memory/lifecycle_tester.cpp rename to src/tests/memory/memory_tester.cpp From c6252078cf7dbd5edf10830f1fdea5cfe80b2436 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 17:42:06 -0600 Subject: [PATCH 09/17] Name alignment --- .github/workflows/ci.yml | 2 +- .github/workflows/tests.yml | 4 +-- docs/testing.md | 22 +++++++------- ...{memory-regression.sh => memory-tester.sh} | 30 +++++++++---------- src/tests/CMakeLists.txt | 14 ++++----- src/tests/memory/memory_tester.cpp | 4 +-- 6 files changed, 38 insertions(+), 38 deletions(-) rename scripts/{memory-regression.sh => memory-tester.sh} (81%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87152589..0b5dd776 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - client-sdk-rust - client-sdk-rust/** - cmake/** - - scripts/memory-regression.sh + - scripts/memory-tester.sh - scripts/set-test-tokens.sh - scripts/run-with-backtrace.sh - CMakeLists.txt diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4d87c24f..1060eeb2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -433,14 +433,14 @@ jobs: --gtest_recreate_environments_when_repeating=1 \ --gtest_output=xml:${{ env.BUILD_DIR }}/integration-test-results.xml - - name: Run memory regression tests + - name: Run memory tester if: matrix.name == 'ubuntu-24.04-x64' && inputs.build_type == 'release' timeout-minutes: 10 shell: bash run: | set -euo pipefail source scripts/set-test-tokens.sh - scripts/memory-regression.sh \ + scripts/memory-tester.sh \ --build-dir "${{ env.BUILD_DIR }}" \ --scenario all \ --iterations 100 \ diff --git a/docs/testing.md b/docs/testing.md index cbf97cc5..f344c837 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -15,7 +15,7 @@ length. | `livekit_unit_tests` | Unit tests (no server required) | | `livekit_integration_tests` | Integration tests (server required) | | `livekit_stress_tests` | Long-running stress tests (server required) | -| `livekit_memory_lifecycle_tester` | Resource leak tester | +| `livekit_memory_tester` | Resource leak tester | ## Building the test binaries @@ -123,11 +123,11 @@ export LIVEKIT_TOKEN_B="$(lk token create --api-key devkey --api-secret secret - - **RPC**: round-trip calls, max payload (15 KB), timeouts, errors, concurrent calls. - **Stress**: high throughput, bidirectional RPC, memory pressure. -## Memory lifecycle tester +## Memory tester > Note: Linux only -The memory lifecycle tester binary catches resource leak regressions in an +The memory tester binary catches resource leak regressions in an automated fashion, and is invoked in CI. This single binary runs individual or groups of scenarios: @@ -146,7 +146,7 @@ Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth and fails on thread growth or more than the configured amount of RSS growth. -The binary can be run directly, or via a helper script at `scripts/memory-regression.sh`, +The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`, which invokes the binary but also has GitHub actions support for a results summary table. The helper defaults `MALLOC_ARENA_MAX` to `1` when launching each tester process. This limits `glibc`'s allocator arenas, reducing Linux thread-churn RSS noise, but it is @@ -158,10 +158,10 @@ tokens before running any scenario: ```bash ./build.sh release-tests source scripts/set-test-tokens.sh -scripts/memory-regression.sh --scenario audio-track \ +scripts/memory-tester.sh --scenario audio-track \ --iterations 100 --warmup 20 \ --max-rss-growth-mib 8 --max-thread-growth 0 -scripts/memory-regression.sh --scenario video-track \ +scripts/memory-tester.sh --scenario video-track \ --iterations 100 --warmup 20 \ --max-rss-growth-mib 8 --max-thread-growth 0 ``` @@ -169,11 +169,11 @@ scripts/memory-regression.sh --scenario video-track \ Use `--scenario all` to run all six scenarios, each in a fresh tester process: ```bash -scripts/memory-regression.sh --scenario data-track -scripts/memory-regression.sh --scenario room-reuse -scripts/memory-regression.sh --scenario room-client-leave -scripts/memory-regression.sh --scenario room-server-delete -scripts/memory-regression.sh --scenario all +scripts/memory-tester.sh --scenario data-track +scripts/memory-tester.sh --scenario room-reuse +scripts/memory-tester.sh --scenario room-client-leave +scripts/memory-tester.sh --scenario room-server-delete +scripts/memory-tester.sh --scenario all ``` The limits are command-line options so a longer local run can use different diff --git a/scripts/memory-regression.sh b/scripts/memory-tester.sh similarity index 81% rename from scripts/memory-regression.sh rename to scripts/memory-tester.sh index cfee8beb..936f48e6 100755 --- a/scripts/memory-regression.sh +++ b/scripts/memory-tester.sh @@ -18,9 +18,9 @@ set -euo pipefail usage() { cat <<'EOF' -Usage: memory-regression.sh [OPTIONS] +Usage: memory-tester.sh [OPTIONS] -Run the Linux memory lifecycle regression tester and append a table to the +Run the Linux memory tester and append a table to the GitHub step summary when GITHUB_STEP_SUMMARY is set. Options: @@ -35,21 +35,21 @@ Options: -h, --help Show this help Environment: - MEMORY_REGRESSION_BUILD_DIR, MEMORY_REGRESSION_SCENARIO, - MEMORY_REGRESSION_ITERATIONS, - MEMORY_REGRESSION_WARMUP, MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB, - MEMORY_REGRESSION_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. + MEMORY_TESTER_BUILD_DIR, MEMORY_TESTER_SCENARIO, + MEMORY_TESTER_ITERATIONS, + MEMORY_TESTER_WARMUP, MEMORY_TESTER_MAX_RSS_GROWTH_MIB, + MEMORY_TESTER_MAX_THREAD_GROWTH, MALLOC_ARENA_MAX. All scenarios require LIVEKIT_URL and LIVEKIT_TOKEN_A. EOF } repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" -build_dir="${MEMORY_REGRESSION_BUILD_DIR:-build-release}" -scenario="${MEMORY_REGRESSION_SCENARIO:-audio-track}" -iterations="${MEMORY_REGRESSION_ITERATIONS:-100}" -warmup="${MEMORY_REGRESSION_WARMUP:-20}" -max_rss_growth_mib="${MEMORY_REGRESSION_MAX_RSS_GROWTH_MIB:-8}" -max_thread_growth="${MEMORY_REGRESSION_MAX_THREAD_GROWTH:-0}" +build_dir="${MEMORY_TESTER_BUILD_DIR:-build-release}" +scenario="${MEMORY_TESTER_SCENARIO:-audio-track}" +iterations="${MEMORY_TESTER_ITERATIONS:-100}" +warmup="${MEMORY_TESTER_WARMUP:-20}" +max_rss_growth_mib="${MEMORY_TESTER_MAX_RSS_GROWTH_MIB:-8}" +max_thread_growth="${MEMORY_TESTER_MAX_THREAD_GROWTH:-0}" while (($#)); do case "$1" in @@ -67,7 +67,7 @@ done if [[ "${build_dir}" != /* ]]; then build_dir="${repo_root}/${build_dir}" fi -tester="${build_dir}/bin/livekit_memory_lifecycle_tester" +tester="${build_dir}/bin/livekit_memory_tester" if [[ ! -x "${tester}" ]]; then echo "ERROR: ${tester} not found. Run ./build.sh release-tests first." >&2 exit 1 @@ -97,7 +97,7 @@ done if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then { - echo "## Memory regression results" + echo "## Memory tester results" echo echo "Iterations: ${iterations}" echo "Warmup: ${warmup}" @@ -108,7 +108,7 @@ if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then } >> "${GITHUB_STEP_SUMMARY}" fi -report_regex='memory lifecycle: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' +report_regex='memory tester: scenario=([a-z-]+), RSS 0 ([0-9]+\.[0-9]+) -> warmup ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\) -> final ([0-9]+\.[0-9]+) MiB \((-?[0-9]+\.[0-9]+) MiB\), threads 0 ([0-9]+) -> warmup ([0-9]+) \((-?[0-9]+)\) -> final ([0-9]+) \((-?[0-9]+)\), verdict=(PASS|FAIL)' format_growth() { if [[ "$1" == "0" || "$1" == "0.00" || "$1" == -* ]]; then diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index 02e02bed..23d5fc65 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -350,20 +350,20 @@ if(TARGET livekit_stress_tests) endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") - add_executable(livekit_memory_lifecycle_tester + add_executable(livekit_memory_tester "${CMAKE_CURRENT_SOURCE_DIR}/memory/memory_tester.cpp" ) - target_link_libraries(livekit_memory_lifecycle_tester PRIVATE livekit) - target_include_directories(livekit_memory_lifecycle_tester PRIVATE + target_link_libraries(livekit_memory_tester PRIVATE livekit) + target_include_directories(livekit_memory_tester PRIVATE ${LIVEKIT_ROOT_DIR}/include ) - add_custom_command(TARGET livekit_memory_lifecycle_tester POST_BUILD + add_custom_command(TARGET livekit_memory_tester POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ - $ + $ COMMAND ${CMAKE_COMMAND} -E copy_if_different "$/liblivekit_ffi.so" - $ - COMMENT "Copying shared libraries next to livekit_memory_lifecycle_tester" + $ + COMMENT "Copying shared libraries next to livekit_memory_tester" ) endif() diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index b8d279ad..ff7b52b0 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -494,7 +494,7 @@ int main(int argc, char* argv[]) { static_cast(final.threads) - static_cast(warmup.threads); const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); - std::cout << std::fixed << std::setprecision(2) << "memory lifecycle: scenario=" << scenarioName(options.scenario) + std::cout << std::fixed << std::setprecision(2) << "memory tester: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << kibToMib(initial.rss_kib) << " -> warmup " << kibToMib(warmup.rss_kib) << " MiB (" << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.rss_kib) << " MiB (" << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.threads << " -> warmup " << warmup.threads @@ -502,7 +502,7 @@ int main(int argc, char* argv[]) { << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { - std::cerr << "memory lifecycle tester failed: " << error.what() << '\n'; + std::cerr << "memory tester failed: " << error.what() << '\n'; return 1; } } From 4cf631b092cf90f8aa1c16980da702a1622a5118 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 17:44:32 -0700 Subject: [PATCH 10/17] Actually catch room server delete --- docs/testing.md | 6 ++++-- src/tests/memory/memory_tester.cpp | 31 ++++++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index f344c837..787a61ed 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -139,8 +139,10 @@ or groups of scenarios: payloads, unpublishes it, and disconnects. - `room-reuse` repeatedly connects and disconnects the same room object. - `room-client-leave` repeatedly connects, disconnects, and destroys a room. -- `room-server-delete` repeatedly connects and uses `lk` to delete the local - test room, then verifies the disconnect and end-of-stream callbacks. +- `room-server-delete` repeatedly connects, publishes one synthetic video + track, captures I420 frames without unpublishing, and uses `lk` to delete + the local test room. It then verifies the disconnect and end-of-stream + callbacks. Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index ff7b52b0..7c4a00f3 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -404,8 +404,11 @@ void runRoomClientLeaveCycle(const RoomCredentials& credentials) { delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); } -// Exercises server-initiated room deletion and validates disconnect, event-stream -// completion, local-participant cleanup, and destruction of the torn-down Room. +// Exercises server-initiated room deletion while one published video track is +// still attached. Teardown must release that publication after the peer +// connection has already closed; unpublishing first avoids the leak this +// scenario targets. The synthetic track uses the default software-capable +// encoder so the scenario can run in CI. void runRoomServerDeleteCycle(const RoomCredentials& credentials) { if (std::string_view(credentials.url) != kLocalTestLiveKitUrl) { throw std::runtime_error("room-server-delete requires LIVEKIT_URL=ws://localhost:7880"); @@ -416,6 +419,30 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { room.setDelegate(&delegate); connectRoom(room, credentials); + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing video track"); + } + + auto video_source = std::make_shared(640, 360); + auto video_track = livekit::LocalVideoTrack::createLocalVideoTrack("memory-video", video_source); + if (!video_track) { + throw std::runtime_error("failed to create local video track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + participant->publishTrack(video_track, publish_options); + if (!video_track->publication()) { + throw std::runtime_error("failed to publish local video track"); + } + + auto frame = livekit::VideoFrame::create(640, 360, livekit::VideoBufferType::I420); + std::fill(frame.data(), frame.data() + frame.dataSize(), 0x7f); + for (int frame_index = 0; frame_index < kFramesPerCycle; ++frame_index) { + video_source->captureFrame(frame); + } + // The room name is fixed rather than interpolating untrusted environment data into the command. if (std::system("lk --dev room delete --yes cpp_data_track_test") != 0) { throw std::runtime_error("failed to delete local test room"); From b3381edbdfd397aeb751ebae33fcffda0a846fba Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 17:48:03 -0700 Subject: [PATCH 11/17] Fix typo --- docs/testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/testing.md b/docs/testing.md index 787a61ed..37254e43 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -44,7 +44,7 @@ After building, test binaries are available at: ## Running tests ```bash -# Run all tests via ctest (warning, runs unit, ) +# Run all tests via ctest (warning, runs all GTest-based tests including stress) cd build-debug ctest --output-on-failure From 565be38ac98650468b867a643972a25de6376542 Mon Sep 17 00:00:00 2001 From: Alan George Date: Thu, 15 Oct 2026 18:35:08 -0700 Subject: [PATCH 12/17] Maybe fix instability --- docs/testing.md | 8 +- src/tests/memory/memory_tester.cpp | 138 ++++++++++++++++++++++++++--- 2 files changed, 130 insertions(+), 16 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 37254e43..81a45e39 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -145,8 +145,12 @@ or groups of scenarios: callbacks. Each scenario reports process RSS and thread count at cycle 0, after warmup -cycle 20, and after final cycle 100. The gate applies to warmup-to-final growth -and fails on thread growth or more than the configured amount of RSS growth. +cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread +count to remain stable for 500 ms so short-lived teardown work does not cause a +false failure. The gate applies to warmup-to-final growth and fails on thread +growth or more than the configured amount of RSS growth. If threads grow, the +tester also reports when growth was first observed and groups the warmup and +final threads by their Linux thread names. The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`, which invokes the binary but also has GitHub actions support for a results summary table. diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index 7c4a00f3..00a99661 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -24,11 +24,14 @@ #include #include #include +#include #include #include #include +#include #include #include +#include #include #include #include @@ -40,6 +43,9 @@ using namespace std::chrono_literals; constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; constexpr int kFramesPerCycle = 3; +constexpr auto kThreadQuiescencePeriod = 500ms; +constexpr auto kThreadQuiescencePollInterval = 50ms; +constexpr auto kThreadQuiescenceTimeout = 2s; enum class Scenario { AudioTrack, @@ -63,6 +69,13 @@ struct ProcessSample { std::uint64_t threads{}; }; +using ThreadNameCounts = std::map; + +struct ProcessSnapshot { + ProcessSample process; + ThreadNameCounts thread_names; +}; + struct RoomCredentials { std::string url; std::string token; @@ -235,6 +248,82 @@ ProcessSample sampleProcess() { return sample; } +ThreadNameCounts sampleThreadNames() { + ThreadNameCounts counts; + std::error_code error; + std::filesystem::directory_iterator iterator("/proc/self/task", error); + const std::filesystem::directory_iterator end; + while (!error && iterator != end) { + std::ifstream comm(iterator->path() / "comm"); + std::string name; + if (std::getline(comm, name) && !name.empty()) { + ++counts[name]; + } + iterator.increment(error); + } + return counts; +} + +ProcessSnapshot sampleProcessSnapshot() { return {sampleProcess(), sampleThreadNames()}; } + +ProcessSnapshot sampleQuiescentProcess() { + auto snapshot = sampleProcessSnapshot(); + auto stable_since = std::chrono::steady_clock::now(); + const auto deadline = stable_since + kThreadQuiescenceTimeout; + while (std::chrono::steady_clock::now() < deadline) { + std::this_thread::sleep_for(kThreadQuiescencePollInterval); + auto next = sampleProcessSnapshot(); + if (next.process.threads != snapshot.process.threads) { + stable_since = std::chrono::steady_clock::now(); + } else if (std::chrono::steady_clock::now() - stable_since >= kThreadQuiescencePeriod) { + return next; + } + snapshot = std::move(next); + } + std::cerr << "memory tester: thread count did not quiesce within " + << std::chrono::duration_cast(kThreadQuiescenceTimeout).count() + << " ms; using the latest sample\n"; + return snapshot; +} + +std::string formatThreadNames(const ThreadNameCounts& counts) { + std::ostringstream output; + bool first = true; + for (const auto& [name, count] : counts) { + if (!first) { + output << ", "; + } + output << name << '=' << count; + first = false; + } + return output.str(); +} + +std::string formatThreadNameDelta(const ThreadNameCounts& before, const ThreadNameCounts& after) { + ThreadNameCounts names = before; + names.insert(after.begin(), after.end()); + + std::ostringstream output; + bool first = true; + for (const auto& [name, unused] : names) { + static_cast(unused); + const auto before_entry = before.find(name); + const auto after_entry = after.find(name); + const auto before_count = before_entry == before.end() ? 0 : before_entry->second; + const auto after_count = after_entry == after.end() ? 0 : after_entry->second; + const auto delta = static_cast(after_count) - static_cast(before_count); + if (delta == 0) { + continue; + } + if (!first) { + output << ", "; + } + output << name << '=' << (delta > 0 ? "+" : "") << delta; + first = false; + } + return output.str(); +} + double kibToMib(std::uint64_t kib) { return static_cast(kib) / 1024.0; } double kibToMib(std::int64_t kib) { return static_cast(kib) / 1024.0; } @@ -499,34 +588,55 @@ int main(int argc, char* argv[]) { reusable_room = std::make_unique(); } - const ProcessSample initial = sampleProcess(); - ProcessSample warmup; - ProcessSample final; + const ProcessSnapshot initial = sampleProcessSnapshot(); + ProcessSnapshot warmup; + ProcessSnapshot final; + std::uint64_t first_thread_growth_iteration = 0; + ProcessSnapshot first_thread_growth; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { runCycle(options.scenario, credentials, reusable_room.get()); std::this_thread::sleep_for(100ms); if (iteration == options.warmup) { - warmup = sampleProcess(); + warmup = sampleQuiescentProcess(); } else if (iteration == options.iterations) { - final = sampleProcess(); + final = sampleQuiescentProcess(); + } else if (iteration > options.warmup && first_thread_growth_iteration == 0) { + auto current = sampleProcessSnapshot(); + if (current.process.threads > warmup.process.threads) { + first_thread_growth_iteration = iteration; + first_thread_growth = std::move(current); + } } } const auto warmup_rss_growth = - static_cast(warmup.rss_kib) - static_cast(initial.rss_kib); - const auto final_rss_growth = static_cast(final.rss_kib) - static_cast(warmup.rss_kib); + static_cast(warmup.process.rss_kib) - static_cast(initial.process.rss_kib); + const auto final_rss_growth = + static_cast(final.process.rss_kib) - static_cast(warmup.process.rss_kib); const auto warmup_thread_growth = - static_cast(warmup.threads) - static_cast(initial.threads); + static_cast(warmup.process.threads) - static_cast(initial.process.threads); const auto final_thread_growth = - static_cast(final.threads) - static_cast(warmup.threads); + static_cast(final.process.threads) - static_cast(warmup.process.threads); const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); + if (final_thread_growth > 0) { + if (first_thread_growth_iteration == 0) { + first_thread_growth_iteration = options.iterations; + first_thread_growth = final; + } + std::cerr << "memory tester: thread growth first observed after cycle " << first_thread_growth_iteration + << "; warmup={" << formatThreadNames(warmup.thread_names) << "}; first-growth={" + << formatThreadNames(first_thread_growth.thread_names) << "}; final={" + << formatThreadNames(final.thread_names) << "}; warmup-to-final delta={" + << formatThreadNameDelta(warmup.thread_names, final.thread_names) << "}\n"; + } std::cout << std::fixed << std::setprecision(2) << "memory tester: scenario=" << scenarioName(options.scenario) - << ", RSS 0 " << kibToMib(initial.rss_kib) << " -> warmup " << kibToMib(warmup.rss_kib) << " MiB (" - << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.rss_kib) << " MiB (" - << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.threads << " -> warmup " << warmup.threads - << " (" << warmup_thread_growth << ") -> final " << final.threads << " (" << final_thread_growth - << "), verdict=" << (passed ? "PASS" : "FAIL") << '\n'; + << ", RSS 0 " << kibToMib(initial.process.rss_kib) << " -> warmup " << kibToMib(warmup.process.rss_kib) + << " MiB (" << kibToMib(warmup_rss_growth) << " MiB) -> final " << kibToMib(final.process.rss_kib) + << " MiB (" << kibToMib(final_rss_growth) << " MiB), threads 0 " << initial.process.threads + << " -> warmup " << warmup.process.threads << " (" << warmup_thread_growth << ") -> final " + << final.process.threads << " (" << final_thread_growth << "), verdict=" << (passed ? "PASS" : "FAIL") + << '\n'; return passed ? 0 : 1; } catch (const std::exception& error) { std::cerr << "memory tester failed: " << error.what() << '\n'; From 205ed52c657ed26ad89912be2873afaaf0976593 Mon Sep 17 00:00:00 2001 From: Alan George Date: Fri, 16 Oct 2026 08:34:59 -0700 Subject: [PATCH 13/17] Address devin.ai comments --- docs/testing.md | 4 +- src/tests/memory/memory_tester.cpp | 63 ++++++++++++++++++------------ 2 files changed, 41 insertions(+), 26 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 81a45e39..eb66f5da 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -149,8 +149,8 @@ cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread count to remain stable for 500 ms so short-lived teardown work does not cause a false failure. The gate applies to warmup-to-final growth and fails on thread growth or more than the configured amount of RSS growth. If threads grow, the -tester also reports when growth was first observed and groups the warmup and -final threads by their Linux thread names. +tester also reports when the final sustained growth began and groups the warmup +and final threads by their Linux thread names. The binary can be run directly, or via a helper script at `scripts/memory-tester.sh`, which invokes the binary but also has GitHub actions support for a results summary table. diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index 00a99661..54db826a 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -70,10 +70,11 @@ struct ProcessSample { }; using ThreadNameCounts = std::map; +using ThreadIdentities = std::map; struct ProcessSnapshot { ProcessSample process; - ThreadNameCounts thread_names; + ThreadIdentities threads; }; struct RoomCredentials { @@ -248,8 +249,8 @@ ProcessSample sampleProcess() { return sample; } -ThreadNameCounts sampleThreadNames() { - ThreadNameCounts counts; +ThreadIdentities sampleThreads() { + ThreadIdentities threads; std::error_code error; std::filesystem::directory_iterator iterator("/proc/self/task", error); const std::filesystem::directory_iterator end; @@ -257,14 +258,14 @@ ThreadNameCounts sampleThreadNames() { std::ifstream comm(iterator->path() / "comm"); std::string name; if (std::getline(comm, name) && !name.empty()) { - ++counts[name]; + threads.emplace(iterator->path().filename().string(), std::move(name)); } iterator.increment(error); } - return counts; + return threads; } -ProcessSnapshot sampleProcessSnapshot() { return {sampleProcess(), sampleThreadNames()}; } +ProcessSnapshot sampleProcessSnapshot() { return {sampleProcess(), sampleThreads()}; } ProcessSnapshot sampleQuiescentProcess() { auto snapshot = sampleProcessSnapshot(); @@ -273,7 +274,7 @@ ProcessSnapshot sampleQuiescentProcess() { while (std::chrono::steady_clock::now() < deadline) { std::this_thread::sleep_for(kThreadQuiescencePollInterval); auto next = sampleProcessSnapshot(); - if (next.process.threads != snapshot.process.threads) { + if (next.threads != snapshot.threads) { stable_since = std::chrono::steady_clock::now(); } else if (std::chrono::steady_clock::now() - stable_since >= kThreadQuiescencePeriod) { return next; @@ -286,10 +287,19 @@ ProcessSnapshot sampleQuiescentProcess() { return snapshot; } -std::string formatThreadNames(const ThreadNameCounts& counts) { +ThreadNameCounts countThreadNames(const ThreadIdentities& threads) { + ThreadNameCounts counts; + for (const auto& [id, name] : threads) { + static_cast(id); + ++counts[name]; + } + return counts; +} + +std::string formatThreadNames(const ThreadIdentities& threads) { std::ostringstream output; bool first = true; - for (const auto& [name, count] : counts) { + for (const auto& [name, count] : countThreadNames(threads)) { if (!first) { output << ", "; } @@ -299,7 +309,9 @@ std::string formatThreadNames(const ThreadNameCounts& counts) { return output.str(); } -std::string formatThreadNameDelta(const ThreadNameCounts& before, const ThreadNameCounts& after) { +std::string formatThreadNameDelta(const ThreadIdentities& before_threads, const ThreadIdentities& after_threads) { + const auto before = countThreadNames(before_threads); + const auto after = countThreadNames(after_threads); ThreadNameCounts names = before; names.insert(after.begin(), after.end()); @@ -591,8 +603,8 @@ int main(int argc, char* argv[]) { const ProcessSnapshot initial = sampleProcessSnapshot(); ProcessSnapshot warmup; ProcessSnapshot final; - std::uint64_t first_thread_growth_iteration = 0; - ProcessSnapshot first_thread_growth; + std::uint64_t sustained_thread_growth_iteration = 0; + ProcessSnapshot sustained_thread_growth; for (std::uint64_t iteration = 1; iteration <= options.iterations; ++iteration) { runCycle(options.scenario, credentials, reusable_room.get()); std::this_thread::sleep_for(100ms); @@ -600,11 +612,14 @@ int main(int argc, char* argv[]) { warmup = sampleQuiescentProcess(); } else if (iteration == options.iterations) { final = sampleQuiescentProcess(); - } else if (iteration > options.warmup && first_thread_growth_iteration == 0) { + } else if (iteration > options.warmup) { auto current = sampleProcessSnapshot(); - if (current.process.threads > warmup.process.threads) { - first_thread_growth_iteration = iteration; - first_thread_growth = std::move(current); + if (current.process.threads <= warmup.process.threads) { + sustained_thread_growth_iteration = 0; + sustained_thread_growth = {}; + } else if (sustained_thread_growth_iteration == 0) { + sustained_thread_growth_iteration = iteration; + sustained_thread_growth = std::move(current); } } } @@ -620,15 +635,15 @@ int main(int argc, char* argv[]) { const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && final_thread_growth <= static_cast(options.max_thread_growth); if (final_thread_growth > 0) { - if (first_thread_growth_iteration == 0) { - first_thread_growth_iteration = options.iterations; - first_thread_growth = final; + if (sustained_thread_growth_iteration == 0) { + sustained_thread_growth_iteration = options.iterations; + sustained_thread_growth = final; } - std::cerr << "memory tester: thread growth first observed after cycle " << first_thread_growth_iteration - << "; warmup={" << formatThreadNames(warmup.thread_names) << "}; first-growth={" - << formatThreadNames(first_thread_growth.thread_names) << "}; final={" - << formatThreadNames(final.thread_names) << "}; warmup-to-final delta={" - << formatThreadNameDelta(warmup.thread_names, final.thread_names) << "}\n"; + std::cerr << "memory tester: final thread growth sustained since cycle " << sustained_thread_growth_iteration + << "; warmup={" << formatThreadNames(warmup.threads) << "}; growth-start={" + << formatThreadNames(sustained_thread_growth.threads) << "}; final={" + << formatThreadNames(final.threads) << "}; warmup-to-final delta={" + << formatThreadNameDelta(warmup.threads, final.threads) << "}\n"; } std::cout << std::fixed << std::setprecision(2) << "memory tester: scenario=" << scenarioName(options.scenario) << ", RSS 0 " << kibToMib(initial.process.rss_kib) << " -> warmup " << kibToMib(warmup.process.rss_kib) From 2845a9878f40ce667b048e216c4738d114a3e552 Mon Sep 17 00:00:00 2001 From: Alan George Date: Mon, 19 Oct 2026 11:20:03 -0700 Subject: [PATCH 14/17] PR feedback --- docs/testing.md | 15 +++------------ scripts/memory-tester.sh | 2 +- src/tests/memory/memory_tester.cpp | 16 +++++++++++++--- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index eb66f5da..48d4bc3c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -31,21 +31,11 @@ length. .\build.cmd release-tests ``` -After building, test binaries are available at: - -```bash -# Linux/macOS -/bin/ - -# Windows -\bin\ -``` - ## Running tests ```bash # Run all tests via ctest (warning, runs all GTest-based tests including stress) -cd build-debug +cd build-debug # or build-release ctest --output-on-failure # Or run test executables directly @@ -183,7 +173,8 @@ scripts/memory-tester.sh --scenario all ``` The limits are command-line options so a longer local run can use different -budgets. +budgets. RSS growth thresholds are expressed in MiB and may be fractional, such +as `--max-rss-growth-mib 0.5`. ## Additional memory checks (valgrind) diff --git a/scripts/memory-tester.sh b/scripts/memory-tester.sh index 936f48e6..89145083 100755 --- a/scripts/memory-tester.sh +++ b/scripts/memory-tester.sh @@ -30,7 +30,7 @@ Options: room-server-delete, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) - --max-rss-growth-mib N Allowed RSS growth (default: 8) + --max-rss-growth-mib MIB Allowed RSS growth; may be fractional (default: 8) --max-thread-growth N Allowed thread growth (default: 0) -h, --help Show this help diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index 54db826a..ceae19b4 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -60,7 +61,7 @@ struct Options { Scenario scenario{Scenario::AudioTrack}; std::uint64_t iterations{100}; std::uint64_t warmup{20}; - std::uint64_t max_rss_growth_mib{8}; + double max_rss_growth_mib{8.0}; std::uint64_t max_thread_growth{0}; }; @@ -191,6 +192,15 @@ std::uint64_t parseUnsigned(std::string_view value, std::string_view option) { return parsed; } +double parseNonnegativeDouble(std::string_view value, std::string_view option) { + double parsed = 0.0; + const auto result = std::from_chars(value.data(), value.data() + value.size(), parsed); + if (result.ec != std::errc{} || result.ptr != value.data() + value.size() || !std::isfinite(parsed) || parsed < 0.0) { + throw std::runtime_error("invalid value for " + std::string(option)); + } + return parsed; +} + Options parseOptions(int argc, char* argv[]) { Options options; for (int index = 1; index < argc; ++index) { @@ -206,7 +216,7 @@ Options parseOptions(int argc, char* argv[]) { } else if (option == "--warmup") { options.warmup = parseUnsigned(value, option); } else if (option == "--max-rss-growth-mib") { - options.max_rss_growth_mib = parseUnsigned(value, option); + options.max_rss_growth_mib = parseNonnegativeDouble(value, option); } else if (option == "--max-thread-growth") { options.max_thread_growth = parseUnsigned(value, option); } else { @@ -632,7 +642,7 @@ int main(int argc, char* argv[]) { static_cast(warmup.process.threads) - static_cast(initial.process.threads); const auto final_thread_growth = static_cast(final.process.threads) - static_cast(warmup.process.threads); - const bool passed = kibToMib(final_rss_growth) <= static_cast(options.max_rss_growth_mib) && + const bool passed = kibToMib(final_rss_growth) <= options.max_rss_growth_mib && final_thread_growth <= static_cast(options.max_thread_growth); if (final_thread_growth > 0) { if (sustained_thread_growth_iteration == 0) { From 16d3f88bf26936f47e7ea4ef0cafb6e38dcc0db3 Mon Sep 17 00:00:00 2001 From: Alan George Date: Fri, 6 Nov 2026 15:36:21 -0800 Subject: [PATCH 15/17] Latest rust --- client-sdk-rust | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client-sdk-rust b/client-sdk-rust index 88705b15..052f0891 160000 --- a/client-sdk-rust +++ b/client-sdk-rust @@ -1 +1 @@ -Subproject commit 88705b15086ecbe2c26ee91ede6203f974a0fc62 +Subproject commit 052f089166b6864bc5f7e92906f6a7897b9f546c From 1234d359155b90d25c188c92b49692cc5c4d80e2 Mon Sep 17 00:00:00 2001 From: Alan George Date: Fri, 6 Nov 2026 16:08:35 -0800 Subject: [PATCH 16/17] Add capture scenarios --- docs/testing.md | 13 +++- scripts/memory-tester.sh | 16 ++++- src/tests/CMakeLists.txt | 3 - src/tests/memory/memory_tester.cpp | 108 ++++++++++++++++++++++++++++- 4 files changed, 131 insertions(+), 9 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index fbce1339..b0bb9434 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -204,6 +204,13 @@ or groups of scenarios: track, captures I420 frames without unpublishing, and uses `lk` to delete the local test room. It then verifies the disconnect and end-of-stream callbacks. +- `capture-clock` repeatedly initializes the SDK, creates and publishes the + GPU-rendered clock capture source, starts and stops its frame pump, joins its + worker, unpublishes, disconnects, and shuts down. Each cycle must produce at + least three frames, so the default 100-cycle run captures at least 300 frames + (10 seconds of video at 30 fps). +- `capture-pattern` exercises the same lifecycle with the GPU-rendered gradient + pattern capture source. Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread @@ -223,7 +230,7 @@ Start `livekit-server --dev`, install the `lk` CLI, and source the local test tokens before running any scenario: ```bash -./build.sh release-tests +./build.sh release-tests --capture=pattern,clock source scripts/set-test-tokens.sh scripts/memory-tester.sh --scenario audio-track \ --iterations 100 --warmup 20 \ @@ -233,13 +240,15 @@ scripts/memory-tester.sh --scenario video-track \ --max-rss-growth-mib 8 --max-thread-growth 0 ``` -Use `--scenario all` to run all six scenarios, each in a fresh tester process: +Use `--scenario all` to run all eight scenarios, each in a fresh tester process: ```bash scripts/memory-tester.sh --scenario data-track scripts/memory-tester.sh --scenario room-reuse scripts/memory-tester.sh --scenario room-client-leave scripts/memory-tester.sh --scenario room-server-delete +scripts/memory-tester.sh --scenario capture-clock +scripts/memory-tester.sh --scenario capture-pattern scripts/memory-tester.sh --scenario all ``` diff --git a/scripts/memory-tester.sh b/scripts/memory-tester.sh index 89145083..672dd547 100755 --- a/scripts/memory-tester.sh +++ b/scripts/memory-tester.sh @@ -27,7 +27,8 @@ Options: --build-dir PATH Build directory (default: build-release) --scenario NAME audio-track, video-track, data-track, room-reuse, room-client-leave, - room-server-delete, or all (default: audio-track) + room-server-delete, capture-clock, + capture-pattern, or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-mib MIB Allowed RSS growth; may be fractional (default: 8) @@ -74,11 +75,20 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete) + audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete|capture-clock|capture-pattern) scenarios=("${scenario}") ;; all) - scenarios=(audio-track video-track data-track room-reuse room-client-leave room-server-delete) + scenarios=( + audio-track + video-track + data-track + room-reuse + room-client-leave + room-server-delete + capture-clock + capture-pattern + ) ;; *) echo "ERROR: invalid scenario: ${scenario}" >&2 diff --git a/src/tests/CMakeLists.txt b/src/tests/CMakeLists.txt index c9db47b0..75109d87 100644 --- a/src/tests/CMakeLists.txt +++ b/src/tests/CMakeLists.txt @@ -367,9 +367,6 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") "${CMAKE_CURRENT_SOURCE_DIR}/memory/memory_tester.cpp" ) target_link_libraries(livekit_memory_tester PRIVATE livekit) - target_include_directories(livekit_memory_tester PRIVATE - ${LIVEKIT_ROOT_DIR}/include - ) add_custom_command(TARGET livekit_memory_tester POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different $ diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index ceae19b4..0c3c8bb4 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -17,6 +17,10 @@ #include #include +#if LIVEKIT_CAPTURE_ENABLED +#include +#endif + #include #include #include @@ -44,6 +48,8 @@ using namespace std::chrono_literals; constexpr std::string_view kLocalTestLiveKitUrl = "ws://localhost:7880"; constexpr int kFramesPerCycle = 3; +constexpr std::uint32_t kCaptureFramerateFps = 30; +constexpr auto kCaptureRunPeriod = 150ms; constexpr auto kThreadQuiescencePeriod = 500ms; constexpr auto kThreadQuiescencePollInterval = 50ms; constexpr auto kThreadQuiescenceTimeout = 2s; @@ -55,6 +61,8 @@ enum class Scenario { RoomReuse, RoomClientLeave, RoomServerDelete, + CaptureClock, + CapturePattern, }; struct Options { @@ -157,6 +165,10 @@ std::string_view scenarioName(Scenario scenario) { return "room-client-leave"; case Scenario::RoomServerDelete: return "room-server-delete"; + case Scenario::CaptureClock: + return "capture-clock"; + case Scenario::CapturePattern: + return "capture-pattern"; } throw std::runtime_error("unknown scenario"); } @@ -180,6 +192,12 @@ Scenario parseScenario(std::string_view value) { if (value == "room-server-delete") { return Scenario::RoomServerDelete; } + if (value == "capture-clock") { + return Scenario::CaptureClock; + } + if (value == "capture-pattern") { + return Scenario::CapturePattern; + } throw std::runtime_error("invalid value for --scenario"); } @@ -567,6 +585,80 @@ void runRoomServerDeleteCycle(const RoomCredentials& credentials) { delegate.requireServerTeardown(); } +#if LIVEKIT_CAPTURE_ENABLED +template +void runCaptureSourceCycle(const RoomCredentials& credentials, Config config, const char* track_name) { + SdkLifetime sdk; + DisconnectTrackingDelegate delegate; + livekit::Room room; + room.setDelegate(&delegate); + connectRoom(room, credentials); + + auto created = livekit::capture::CaptureSource::create(std::move(config)).get(); + if (!created) { + throw std::runtime_error("failed to create capture source: " + created.error().message); + } + auto capture = std::move(created).value(); + auto track = livekit::LocalVideoTrack::createLocalVideoTrack(track_name, capture->videoSource()); + if (!track) { + throw std::runtime_error("failed to create capture video track"); + } + + auto participant = room.localParticipant().lock(); + if (!participant) { + throw std::runtime_error("local participant expired before publishing capture track"); + } + livekit::TrackPublishOptions publish_options; + publish_options.source = livekit::TrackSource::SOURCE_CAMERA; + publish_options.simulcast = false; + participant->publishTrack(track, capture->publishOptions(publish_options)); + const auto publication = track->publication(); + if (!publication) { + throw std::runtime_error("failed to publish capture video track"); + } + + const auto started = capture->start(); + if (!started) { + throw std::runtime_error("failed to start capture source: " + started.error().message); + } + std::this_thread::sleep_for(kCaptureRunPeriod); + const auto stopped = capture->stop(); + if (!stopped) { + throw std::runtime_error("failed to stop capture source: " + stopped.error().message); + } + const auto finished = capture->join().get(); + if (!finished) { + throw std::runtime_error("capture source failed: " + finished.error().message); + } + if (finished.value().exit != livekit::capture::CaptureExit::Stopped || + finished.value().frames_captured < static_cast(kFramesPerCycle)) { + throw std::runtime_error("capture source stopped before producing the minimum frame count"); + } + + participant->unpublishTrack(publication->sid()); + if (!room.disconnect()) { + throw std::runtime_error("client room disconnect failed"); + } + delegate.requireDisconnect(livekit::DisconnectReason::ClientInitiated); +} + +// Exercises clock-source creation, publication, pump synchronization, +// encoder/resource teardown, and complete SDK shutdown. +void runCaptureClockCycle(const RoomCredentials& credentials) { + runCaptureSourceCycle(credentials, livekit::capture::ClockVideoSourceConfig{{640, 360}, kCaptureFramerateFps}, + "memory-capture-clock"); +} + +// Exercises pattern-source creation, publication, pump synchronization, +// encoder/resource teardown, and complete SDK shutdown. +void runCapturePatternCycle(const RoomCredentials& credentials) { + runCaptureSourceCycle( + credentials, + livekit::capture::PatternVideoSourceConfig{{640, 360}, kCaptureFramerateFps, livekit::capture::Pattern::Gradient}, + "memory-capture-pattern"); +} +#endif + void runCycle(Scenario scenario, const RoomCredentials& credentials, livekit::Room* reusable_room) { switch (scenario) { case Scenario::AudioTrack: @@ -590,6 +682,18 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials, livekit::Ro case Scenario::RoomServerDelete: runRoomServerDeleteCycle(credentials); return; +#if LIVEKIT_CAPTURE_ENABLED + case Scenario::CaptureClock: + runCaptureClockCycle(credentials); + return; + case Scenario::CapturePattern: + runCapturePatternCycle(credentials); + return; +#else + case Scenario::CaptureClock: + case Scenario::CapturePattern: + throw std::runtime_error("capture scenarios require a capture-enabled SDK build"); +#endif } throw std::runtime_error("unknown scenario"); } @@ -600,7 +704,9 @@ int main(int argc, char* argv[]) { try { const Options options = parseOptions(argc, argv); const RoomCredentials credentials = roomCredentials(); - const bool per_cycle_sdk = options.scenario == Scenario::AudioTrack || options.scenario == Scenario::VideoTrack; + const bool per_cycle_sdk = options.scenario == Scenario::AudioTrack || options.scenario == Scenario::VideoTrack || + options.scenario == Scenario::CaptureClock || + options.scenario == Scenario::CapturePattern; std::unique_ptr room_sdk; if (!per_cycle_sdk) { room_sdk = std::make_unique(); From 4e1ac0b5ffc1594f2048fa8a303698aa084f66c0 Mon Sep 17 00:00:00 2001 From: Alan George Date: Fri, 6 Nov 2026 16:33:59 -0800 Subject: [PATCH 17/17] Add gstreamer capture test --- docs/testing.md | 8 ++++++-- scripts/memory-tester.sh | 6 ++++-- src/tests/memory/memory_tester.cpp | 28 +++++++++++++++++++++++++++- 3 files changed, 37 insertions(+), 5 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index b0bb9434..df0faf3c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -211,6 +211,9 @@ or groups of scenarios: (10 seconds of video at 30 fps). - `capture-pattern` exercises the same lifecycle with the GPU-rendered gradient pattern capture source. +- `capture-gstreamer` exercises the same lifecycle through an entirely + software-based GStreamer pipeline (`videotestsrc` and `vp8enc`), including + encoded-frame ingest and WebRTC passthrough. Each scenario reports process RSS and thread count at cycle 0, after warmup cycle 20, and after final cycle 100. Checkpoint sampling waits for the thread @@ -230,7 +233,7 @@ Start `livekit-server --dev`, install the `lk` CLI, and source the local test tokens before running any scenario: ```bash -./build.sh release-tests --capture=pattern,clock +./build.sh release-tests --capture=all source scripts/set-test-tokens.sh scripts/memory-tester.sh --scenario audio-track \ --iterations 100 --warmup 20 \ @@ -240,7 +243,7 @@ scripts/memory-tester.sh --scenario video-track \ --max-rss-growth-mib 8 --max-thread-growth 0 ``` -Use `--scenario all` to run all eight scenarios, each in a fresh tester process: +Use `--scenario all` to run all nine scenarios, each in a fresh tester process: ```bash scripts/memory-tester.sh --scenario data-track @@ -249,6 +252,7 @@ scripts/memory-tester.sh --scenario room-client-leave scripts/memory-tester.sh --scenario room-server-delete scripts/memory-tester.sh --scenario capture-clock scripts/memory-tester.sh --scenario capture-pattern +scripts/memory-tester.sh --scenario capture-gstreamer scripts/memory-tester.sh --scenario all ``` diff --git a/scripts/memory-tester.sh b/scripts/memory-tester.sh index 672dd547..6eb6e9ef 100755 --- a/scripts/memory-tester.sh +++ b/scripts/memory-tester.sh @@ -28,7 +28,8 @@ Options: --scenario NAME audio-track, video-track, data-track, room-reuse, room-client-leave, room-server-delete, capture-clock, - capture-pattern, or all (default: audio-track) + capture-pattern, capture-gstreamer, + or all (default: audio-track) --iterations N Lifecycle cycles (default: 100) --warmup N Baseline cycle (default: 20) --max-rss-growth-mib MIB Allowed RSS growth; may be fractional (default: 8) @@ -75,7 +76,7 @@ if [[ ! -x "${tester}" ]]; then fi case "${scenario}" in - audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete|capture-clock|capture-pattern) + audio-track|video-track|data-track|room-reuse|room-client-leave|room-server-delete|capture-clock|capture-pattern|capture-gstreamer) scenarios=("${scenario}") ;; all) @@ -88,6 +89,7 @@ case "${scenario}" in room-server-delete capture-clock capture-pattern + capture-gstreamer ) ;; *) diff --git a/src/tests/memory/memory_tester.cpp b/src/tests/memory/memory_tester.cpp index 0c3c8bb4..99325a14 100644 --- a/src/tests/memory/memory_tester.cpp +++ b/src/tests/memory/memory_tester.cpp @@ -63,6 +63,7 @@ enum class Scenario { RoomServerDelete, CaptureClock, CapturePattern, + CaptureGstreamer, }; struct Options { @@ -169,6 +170,8 @@ std::string_view scenarioName(Scenario scenario) { return "capture-clock"; case Scenario::CapturePattern: return "capture-pattern"; + case Scenario::CaptureGstreamer: + return "capture-gstreamer"; } throw std::runtime_error("unknown scenario"); } @@ -198,6 +201,9 @@ Scenario parseScenario(std::string_view value) { if (value == "capture-pattern") { return Scenario::CapturePattern; } + if (value == "capture-gstreamer") { + return Scenario::CaptureGstreamer; + } throw std::runtime_error("invalid value for --scenario"); } @@ -657,6 +663,21 @@ void runCapturePatternCycle(const RoomCredentials& credentials) { livekit::capture::PatternVideoSourceConfig{{640, 360}, kCaptureFramerateFps, livekit::capture::Pattern::Gradient}, "memory-capture-pattern"); } + +// Exercises a software-only GStreamer producer and encoder, encoded-frame +// ingest, publication, pump synchronization, and complete SDK shutdown. +void runCaptureGstreamerCycle(const RoomCredentials& credentials) { + livekit::capture::GstreamerVideoSourceConfig config; + config.pipeline = + "videotestsrc is-live=true ! video/x-raw,width=640,height=360,framerate=30/1 ! " + "vp8enc name=lk_encoder deadline=1 target-bitrate=500000 ! " + "appsink name=lk_appsink max-buffers=2 drop=true"; + config.codec = livekit::VideoCodec::VP8; + config.resolution = livekit::capture::CaptureResolution{640, 360}; + config.rate_control = livekit::capture::GstreamerRateControl{"lk_encoder", "target-bitrate", + livekit::capture::GstreamerBitrateUnit::Bps}; + runCaptureSourceCycle(credentials, std::move(config), "memory-capture-gstreamer"); +} #endif void runCycle(Scenario scenario, const RoomCredentials& credentials, livekit::Room* reusable_room) { @@ -689,9 +710,13 @@ void runCycle(Scenario scenario, const RoomCredentials& credentials, livekit::Ro case Scenario::CapturePattern: runCapturePatternCycle(credentials); return; + case Scenario::CaptureGstreamer: + runCaptureGstreamerCycle(credentials); + return; #else case Scenario::CaptureClock: case Scenario::CapturePattern: + case Scenario::CaptureGstreamer: throw std::runtime_error("capture scenarios require a capture-enabled SDK build"); #endif } @@ -706,7 +731,8 @@ int main(int argc, char* argv[]) { const RoomCredentials credentials = roomCredentials(); const bool per_cycle_sdk = options.scenario == Scenario::AudioTrack || options.scenario == Scenario::VideoTrack || options.scenario == Scenario::CaptureClock || - options.scenario == Scenario::CapturePattern; + options.scenario == Scenario::CapturePattern || + options.scenario == Scenario::CaptureGstreamer; std::unique_ptr room_sdk; if (!per_cycle_sdk) { room_sdk = std::make_unique();