From 5b32cc4a62b7e88331ad5b2a87b66b5d16c8081b Mon Sep 17 00:00:00 2001 From: Ahnaf An Nafee Date: Thu, 1 Oct 2026 05:24:06 -0400 Subject: [PATCH] refactor(tools)!: rename ninfer-switch to llm-switch The switcher also runs llama.cpp and Strata servers from the profile overlay, so the name now says what it manages: one local LLM server on the GPU at a time. Commands and the profile format are unchanged, and the docs describe which servers fit (any that takes --host and --port and answers GET /health). BREAKING CHANGE: the command is llm-switch; its overlay and override variables are LLM_SWITCH_PROFILES, LLM_SWITCH_MODELS and LLM_SWITCH_PORT (were NINFER_PROFILES, NINFER_MODELS and NINFER_PORT); its state file and log moved from %LOCALAPPDATA%\ninfer to %LOCALAPPDATA%\llm-switch. Rename the variables, and move switch-state.json to keep the remembered profile. --- README.md | 36 ++++++++++--------- docs/rtx-3080-precision.md | 4 +-- tools/dsh/README.md | 2 +- tools/windows/llm-switch.cmd | 9 +++++ .../{ninfer-switch.ps1 => llm-switch.ps1} | 25 ++++++------- tools/windows/ninfer-switch.cmd | 9 ----- 6 files changed, 44 insertions(+), 41 deletions(-) create mode 100644 tools/windows/llm-switch.cmd rename tools/windows/{ninfer-switch.ps1 => llm-switch.ps1} (91%) delete mode 100644 tools/windows/ninfer-switch.cmd diff --git a/README.md b/README.md index 98590d52..c37fe60d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ other GPUs and the complete engine reference, see the - [Benchmarks](#benchmarks) - [Quick start](#quick-start) - [Profiles](#profiles) -- [ninfer-switch](#ninfer-switch) +- [llm-switch](#llm-switch) - [Connecting a client](#connecting-a-client) - [Building](#building) - [VRAM on a desktop card](#vram-on-a-desktop-card) @@ -118,7 +118,7 @@ MTP in llama.cpp made the ternary models slower on this card (58 down to 43-51 t .\scripts\build.ps1 ``` -2. **Get the model** into `models\`, or point `NINFER_MODELS` at its folder: +2. **Get the model** into `models\`, or point `LLM_SWITCH_MODELS` at its folder: ```powershell hf download emiltsoi/Ternary-Bonsai-2-27B-Uncensored-Heretic-NInfer --local-dir models @@ -127,7 +127,7 @@ MTP in llama.cpp made the ternary models slower on this card (58 down to 43-51 t 3. **Serve it.** Add `tools\windows` to `PATH`, then: ```text - ninfer-switch reason64 + llm-switch reason64 ``` The server listens on `http://127.0.0.1:8080/v1` and speaks both the OpenAI and the Anthropic API. @@ -163,27 +163,29 @@ checks, but two answers still contained invalid generated tests. These changes i precision and mitigate observed overthinking; they do not establish a universal looping fix. See the guide's [reliability findings](docs/rtx-3080-precision.md#reasoning-reliability). -`ninfer-switch start` remembers the last profile; when no saved choice exists it selects -`reason64`. Select `ninfer-switch reason64` once to replace a previously remembered profile. +`llm-switch start` remembers the last profile; when no saved choice exists it selects +`reason64`. Select `llm-switch reason64` once to replace a previously remembered profile. -## ninfer-switch +## llm-switch -Start, switch, stop and inspect the server from any terminal. The server runs hidden and outlives the -window that started it. +Start, switch, stop and inspect the local LLM server from any terminal. One server holds the GPU at a +time, whichever engine it uses. The server runs hidden and outlives the window that started it. | Command | Does | |---|---| -| `ninfer-switch ` | start a profile, stopping whatever runs now | -| `ninfer-switch status` | profile, uptime, port owner, served model, GPU memory, KV capacity, last rate | -| `ninfer-switch logs [-f]` | show or follow the server log | -| `ninfer-switch kill` | stop the server | -| `ninfer-switch start` | start the last profile again | -| `ninfer-switch list` | list the profiles | +| `llm-switch ` | start a profile, stopping whatever runs now | +| `llm-switch status` | profile, uptime, port owner, served model, GPU memory, KV capacity, last rate | +| `llm-switch logs [-f]` | show or follow the server log | +| `llm-switch kill` | stop the server | +| `llm-switch start` | start the last profile again | +| `llm-switch list` | list the profiles | - **Short on VRAM?** `reason64` refuses to start rather than shrinking below its 64K minimum. Other profiles retry in 8K steps down to their `minimumContext`, or 16K when none is specified. -- **Your own profiles.** A second file named by `NINFER_PROFILES` adds or replaces profiles and can - set `"port"` and `"models"`, so servers of other engines fit behind the same command. +- **Your own profiles.** A second file named by `LLM_SWITCH_PROFILES` adds or replaces profiles and + can set `"port"` and `"models"`. Any server that takes `--host` and `--port` and answers + `GET /health` fits, such as llama.cpp's `llama-server` or [Strata](https://github.com/Niko1221/Strata)'s + `serve/server.py`. - **Safe on a shared port.** It only stops server binaries it can start; anything else on the port is reported and left alone. @@ -248,7 +250,7 @@ system RAM and decode fell by 30 to 90% with no error at all. Two things help: | `scripts/build.ps1` finds VS 2022, CUDA 13.x and vcpkg | it never looked in `Program Files` for VS 2022 IDE editions, knew only CUDA 12.8 and 12.9, and passed no vcpkg toolchain | | Built-in RTX 3080 device profile | the card starts on measured routes instead of calibrating for 76 s on first launch | | `--kv-headroom-mib` listed in `--help` | `--kv-capacity auto` holds back 1 GiB by default, which a 10 GB card cannot spare | -| `tools/windows/ninfer-switch` | start, switch, stop and inspect profiles on Windows | +| `tools/windows/llm-switch` | start, switch, stop and inspect local LLM servers on Windows | | CI on checkout v7, setup-node v7 and Node 24 | the Linux script guard passes again, and nothing runs on the deprecated Node 20 runtime | ## Credits and license diff --git a/docs/rtx-3080-precision.md b/docs/rtx-3080-precision.md index 0562e4e2..c82fdbb5 100644 --- a/docs/rtx-3080-precision.md +++ b/docs/rtx-3080-precision.md @@ -104,7 +104,7 @@ lost when the model's weights were originally quantized. Start the profile from this checkout so the switcher reads the updated profile file: ```powershell -.\tools\windows\ninfer-switch.ps1 precision16 +.\tools\windows\llm-switch.ps1 precision16 ``` Reproduce the accuracy runs from the repository root, changing the cache and output directory: @@ -165,7 +165,7 @@ standalone assertion tests. This small experiment does not isolate the source of but it supplies no basis for treating removal of INT8 activation rounding as a general cure. ```powershell -.\tools\windows\ninfer-switch.ps1 reason64 +.\tools\windows\llm-switch.ps1 reason64 ``` The short-context diagnostic profile additionally uses FP32 recurrent-state storage and the corrected diff --git a/tools/dsh/README.md b/tools/dsh/README.md index bdd29d22..00666d70 100644 --- a/tools/dsh/README.md +++ b/tools/dsh/README.md @@ -56,7 +56,7 @@ After the adapter and stats registrations are installed, later module updates on copy. Restart DSH to load the updated modules; use `dsh web --port 3081 --no-open` to retain the existing browser tab without opening another. -Start NInfer with `./tools/windows/ninfer-switch.ps1 reason64`. That profile supplies the 65,536-token +Start NInfer with `./tools/windows/llm-switch.ps1 reason64`. That profile supplies the 65,536-token minimum, RK4 KV, FP32 recurrent state, low thinking capped at 2,048 tokens per step, temperature 0.2, and 8,192 maximum output tokens. It unloads the MTP draft head to make room for the more accurate cache. The client discovers actual server capacity; its preset does not allocate GPU context. diff --git a/tools/windows/llm-switch.cmd b/tools/windows/llm-switch.cmd new file mode 100644 index 00000000..1c5d1461 --- /dev/null +++ b/tools/windows/llm-switch.cmd @@ -0,0 +1,9 @@ +@echo off +rem Lets llm-switch run from cmd, PowerShell or the Run dialog once this folder is on PATH. +where pwsh >nul 2>nul +if %errorlevel%==0 ( + pwsh -NoProfile -ExecutionPolicy Bypass -File "%~dp0llm-switch.ps1" %* +) else ( + powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0llm-switch.ps1" %* +) +exit /b %errorlevel% diff --git a/tools/windows/ninfer-switch.ps1 b/tools/windows/llm-switch.ps1 similarity index 91% rename from tools/windows/ninfer-switch.ps1 rename to tools/windows/llm-switch.ps1 index ecd07fa6..67fbfb2d 100644 --- a/tools/windows/ninfer-switch.ps1 +++ b/tools/windows/llm-switch.ps1 @@ -1,25 +1,26 @@ -# ninfer-switch |start|restart|status|kill|logs [-f]|list +# llm-switch |start|restart|status|kill|logs [-f]|list # -# Windows counterpart of a systemd-driven switcher: there is no service manager here, so the server +# Runs one local LLM server at a time on the GPU: NInfer, llama.cpp, Strata, or any server that +# takes --host and --port and answers GET /health. Windows has no service manager, so the server # runs as a hidden background process that outlives the terminal, and the choice is remembered in a # state file so `start` and `restart` bring back the last profile. # -# Profiles come from profiles.json next to this script. NINFER_PROFILES names a second file whose +# Profiles come from profiles.json next to this script. LLM_SWITCH_PROFILES names a second file whose # profiles are added on top (same name replaces), and whose "models" and "port" override the # defaults. In exe and args, {repo} is this checkout, {models} is the models directory -# (NINFER_MODELS, else the overlay's "models", else {repo}\models), and {here} is the directory of +# (LLM_SWITCH_MODELS, else the overlay's "models", else {repo}\models), and {here} is the directory of # the file that defines the profile. --host and --port are appended, so profiles never repeat them. param([Parameter(Position = 0)][string]$Command = 'help', [Parameter(Position = 1)][string]$Arg) $ErrorActionPreference = 'Stop' $Repo = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) -$StateDir = Join-Path $env:LOCALAPPDATA 'ninfer' +$StateDir = Join-Path $env:LOCALAPPDATA 'llm-switch' $StateFile = Join-Path $StateDir 'switch-state.json' $Log = Join-Path $StateDir 'serve.log' function Read-Profiles { $files = @(Join-Path $PSScriptRoot 'profiles.json') - if ($env:NINFER_PROFILES) { $files += $env:NINFER_PROFILES } + if ($env:LLM_SWITCH_PROFILES) { $files += $env:LLM_SWITCH_PROFILES } $cfg = @{ port = 8080; models = (Join-Path $Repo 'models'); profiles = [ordered]@{} } foreach ($f in $files) { if (-not (Test-Path -LiteralPath $f)) { throw "profiles file not found: $f" } @@ -31,8 +32,8 @@ function Read-Profiles { $cfg.profiles[$p.Name] = $p.Value } } - if ($env:NINFER_MODELS) { $cfg.models = $env:NINFER_MODELS } - if ($env:NINFER_PORT) { $cfg.port = [int]$env:NINFER_PORT } + if ($env:LLM_SWITCH_MODELS) { $cfg.models = $env:LLM_SWITCH_MODELS } + if ($env:LLM_SWITCH_PORT) { $cfg.port = [int]$env:LLM_SWITCH_PORT } $cfg } @@ -77,7 +78,7 @@ function Stop-Server($cfg) { function Start-Server([string]$name, $cfg) { $p = $cfg.profiles[$name] - if (-not $p) { throw "unknown profile '$name'. Run: ninfer-switch list" } + if (-not $p) { throw "unknown profile '$name'. Run: llm-switch list" } $exe = Expand $p.exe $cfg $p $argv = @($p.args | ForEach-Object { Expand $_ $cfg $p }) + @('--host', '127.0.0.1', '--port', "$($cfg.port)") if (-not (Test-Path -LiteralPath $exe)) { throw "server binary not found: $exe" } @@ -93,7 +94,7 @@ function Start-Server([string]$name, $cfg) { if (Stop-Server $cfg) { Write-Host 'stopped the running server' } $o = Get-Owner $cfg if ($o -and -not $o.Ours) { - throw "port $($cfg.port) is taken by $($o.Process.ProcessName) (pid $($o.Process.Id)); set NINFER_PORT or ""port"" in profiles" + throw "port $($cfg.port) is taken by $($o.Process.ProcessName) (pid $($o.Process.Id)); set LLM_SWITCH_PORT or ""port"" in profiles" } New-Item -ItemType Directory -Force -Path $StateDir | Out-Null Write-Host "starting $name ($($p.note))" @@ -148,7 +149,7 @@ function Show-Status($cfg) { $server = if ($ours) { $o.Process } else { $proc } $since = if ($state) { [datetime]$state.started } else { $server.StartTime } $up = (Get-Date) - $since - $name = if ($state) { $state.profile } else { 'unknown (not started by ninfer-switch)' } + $name = if ($state) { $state.profile } else { 'unknown (not started by llm-switch)' } $note = if ($state -and $cfg.profiles.Contains($state.profile)) { " ($($cfg.profiles[$state.profile].note))" } "profile : $name$note" "process : $($server.ProcessName) pid $($server.Id), up {0:%d}d {0:hh}h {0:mm}m (since {1:yyyy-MM-dd HH:mm})" -f $up, $since @@ -197,7 +198,7 @@ switch ($Command) { break } { $_ -in 'help', 'list', '-h', '--help' } { - 'usage: ninfer-switch |start [profile]|restart|status|kill|logs [-f]|list' + 'usage: llm-switch |start [profile]|restart|status|kill|logs [-f]|list' '' 'profiles:' foreach ($k in $cfg.profiles.Keys) { ' {0,-8} {1}' -f $k, $cfg.profiles[$k].note } diff --git a/tools/windows/ninfer-switch.cmd b/tools/windows/ninfer-switch.cmd deleted file mode 100644 index 98912d3c..00000000 --- a/tools/windows/ninfer-switch.cmd +++ /dev/null @@ -1,9 +0,0 @@ -@echo off -rem Lets ninfer-switch run from cmd, PowerShell or the Run dialog once this folder is on PATH. -where pwsh >nul 2>nul -if %errorlevel%==0 ( - pwsh -NoProfile -ExecutionPolicy Bypass -File "%~dp0ninfer-switch.ps1" %* -) else ( - powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0ninfer-switch.ps1" %* -) -exit /b %errorlevel%