diff --git a/assets/voicetranser/icon.png b/assets/voicetranser/icon.png new file mode 100644 index 00000000..d5bed0b3 Binary files /dev/null and b/assets/voicetranser/icon.png differ diff --git a/external_plugins/voicetranser/.zcode-plugin/plugin.json b/external_plugins/voicetranser/.zcode-plugin/plugin.json new file mode 100644 index 00000000..88805c3e --- /dev/null +++ b/external_plugins/voicetranser/.zcode-plugin/plugin.json @@ -0,0 +1,29 @@ +{ + "name": "voicetranser", + "version": "1.0.3", + "description": "Local push-to-talk voice input for macOS: hold a global hotkey, speak, release — transcription is pasted at your cursor. Fully local STT (Qwen3-ASR MLX, SenseVoice, mlx-whisper), zero external APIs. The bundled skill guides your agent through install, hotkey wiring, launchd setup and troubleshooting.", + "description_i18n": { + "en": "Local push-to-talk voice input for macOS: hold a global hotkey, speak, release — transcription is pasted at your cursor. Fully local STT (Qwen3-ASR MLX, SenseVoice, mlx-whisper), zero external APIs. The bundled skill guides your agent through install, hotkey wiring, launchd setup and troubleshooting.", + "zh-CN": "macOS 本地按住说话语音输入：按住全局热键说话、松开即转写粘贴到光标处。纯本地 STT（Qwen3-ASR MLX / SenseVoice / mlx-whisper），零外部 API。内置技能引导 agent 完成安装、热键接线、launchd 常驻与排障。" + }, + "author": { + "name": "Lumos-789", + "url": "https://github.com/Lumos-789" + }, + "homepage": "https://github.com/Lumos-789/VoiceTranser", + "repository": "https://github.com/Lumos-789/VoiceTranser", + "license": "MIT", + "keywords": [ + "voice-input", + "speech-to-text", + "dictation", + "macos", + "push-to-talk", + "local-first", + "qwen3-asr", + "sensevoice" + ], + "skills": [ + "skills/voicetranser" + ] +} diff --git a/external_plugins/voicetranser/README.md b/external_plugins/voicetranser/README.md new file mode 100644 index 00000000..08a44d1c --- /dev/null +++ b/external_plugins/voicetranser/README.md @@ -0,0 +1,38 @@ +# VoiceTranser + +Local push-to-talk voice input for macOS: hold a global hotkey, speak, release — the transcription is pasted at your cursor. Built for talking to coding agents (and anything else you type into). Fully local, zero external APIs. + +This plugin ships an **agent skill** — it does not bundle the tool itself. Once installed, you can say *"install VoiceTranser for me"* and the agent walks the whole chain: prerequisites, clone, deps, model download, launchd service, Karabiner hotkey wiring, accessibility permission, then verifies with the built-in 8-point diagnose script. It also handles ongoing troubleshooting and engine switching. + +The runtime is pulled from the upstream repository, so installation needs network access and the tools listed below. The launchd step generates a plist from the current checkout path with `install_launchd.py`; the repository ships no plist snapshot to copy, so the generated one is the only source. + +## What you get + +- 🎙️ **Push-to-talk** — Karabiner `right_option` hold-to-talk (or Raycast/Hammerspoon hitting `/toggle`) +- 🧠 **Local STT, three engines** — Qwen3-ASR 0.6B 4bit on MLX/Metal (default, RTF 0.012–0.021, excellent Chinese with punctuation, strong zh-en code-switching) → SenseVoice Small int8 (CPU fallback) → mlx-whisper large-v3 (English/exotic languages) +- 📋 **Auto-paste** at the cursor, macOS notifications at each stage +- 🔧 **Mis-recognition wordlist** (`corrections.json`) — add a line, restart, fixed +- 🛡️ **Self-healing** — launchd KeepAlive + CoreAudio watchdog + +## Requirements + +macOS **Apple Silicon** (the default engine needs MLX/Metal; Intel Macs can force the CPU engine), Python 3.11+, [uv](https://docs.astral.sh/uv/). Optional but recommended: [Karabiner-Elements](https://karabiner-elements.pqrs.org/). + +## Example prompts + +- "Install VoiceTranser and wire my right Option key for push-to-talk" +- "语音输入装一下，右 Option 按住说话" +- "Transcription pasted nothing — diagnose VoiceTranser" +- "把 VoiceTranser 切到 CPU 引擎" + +## Side effects (disclosed) + +Installing/configuring VoiceTranser via this skill will, on your machine: + +- **Network**: clone the GitHub repo, install Python deps via uv, download STT models (~230 MB SenseVoice from hf-mirror; ~500 MB Qwen3-ASR auto-fetched on first run). No audio or transcripts ever leaves the machine. +- **Files**: clone the repo, write model files and `~/.cache/huggingface` entries, install a launchd plist into `~/Library/LaunchAgents/`, write logs to `~/Library/Logs/`. +- **Commands**: run `uv`, `launchctl`, `curl` against `localhost:9877`, and the repo's diagnose script. May ask you to grant **Accessibility** permission to your terminal (required for auto-paste) and to add a Karabiner rule (the skill asks before editing Karabiner config or installing brew packages). + +## Source & license + +Upstream project: https://github.com/Lumos-789/VoiceTranser — MIT. diff --git a/external_plugins/voicetranser/README_CN.md b/external_plugins/voicetranser/README_CN.md new file mode 100644 index 00000000..3f762d57 --- /dev/null +++ b/external_plugins/voicetranser/README_CN.md @@ -0,0 +1,38 @@ +# VoiceTranser + +macOS 本地按住说话语音输入：按住全局热键说话、松开即转写粘贴到光标处。为「跟编程助手对话」以及一切打字场景而生。完全本地运行，零外部 API。 + +本插件提供的是 **agent 技能**，不内嵌工具本体。安装后对 agent 说「帮我装 VoiceTranser」，它就会走完整条链：前置检查、clone、依赖、模型下载、launchd 常驻服务、Karabiner 热键接线、辅助功能权限，最后用内置 8 项自检脚本验证收工（音频项有 3 秒硬超时、不会弹麦克风授权；无权限时明确报「未授权」并计为未完整通过，可用 `--skip-audio` 跳过）。日常排障与引擎切换也由它接管。 + +工具本体从上游仓库拉取，因此安装需要网络和下面列出的依赖。launchd 步骤会用 `install_launchd.py` 按当前 checkout 路径生成 plist；仓库不再保存 plist 快照，生成出来的那份就是唯一来源。 + +## 你会得到什么 + +- 🎙️ **按住说话** — Karabiner `right_option` 按住录音松开出稿（也可用 Raycast/Hammerspoon 触发 `/toggle`） +- 🧠 **本地 STT，三引擎** — Qwen3-ASR 0.6B 4bit MLX/Metal（默认，RTF 0.012–0.021，中文近完美自带标点、中英夹杂强）→ SenseVoice Small int8（CPU fallback）→ mlx-whisper large-v3（英文/小语种） +- 📋 **自动粘贴**到光标处，各阶段 macOS 通知反馈 +- 🔧 **误识词表**（`corrections.json`）— 发现误识加一行重启即生效 +- 🛡️ **自愈守护** — launchd KeepAlive + CoreAudio 死锁看门狗 + +## 环境要求 + +macOS **Apple Silicon**（默认引擎需要 MLX/Metal；Intel 机型可强制走 CPU 引擎）、Python 3.11+、[uv](https://docs.astral.sh/uv/)。推荐装 [Karabiner-Elements](https://karabiner-elements.pqrs.org/)。 + +## 示例指令 + +- "Install VoiceTranser and wire my right Option key for push-to-talk" +- "语音输入装一下，右 Option 按住说话" +- "Transcription pasted nothing — diagnose VoiceTranser" +- "把 VoiceTranser 切到 CPU 引擎" + +## 副作用披露 + +通过本技能安装/配置 VoiceTranser 会在你的机器上： + +- **网络**：clone GitHub 仓库、uv 安装 Python 依赖、下载 STT 模型（SenseVoice ~230MB 走 hf-mirror；Qwen3-ASR ~500MB 首次运行自动拉取）。音频与转写文本永不离开本机。 +- **文件**：clone 仓库、写模型文件与 `~/.cache/huggingface` 缓存、安装 launchd plist 到 `~/Library/LaunchAgents/`、日志写到 `~/Library/Logs/`。 +- **命令**：运行 `uv`、`launchctl`、对 `localhost:9877` 的 `curl`、仓库自检脚本；可能需要你为终端授予**辅助功能**权限（自动粘贴必需），并添加一条 Karabiner 规则（改 Karabiner 配置或装 brew 包之前技能会先征询）。 + +## 源码与许可 + +上游项目：https://github.com/Lumos-789/VoiceTranser — MIT。 diff --git a/external_plugins/voicetranser/skills/voicetranser/SKILL.md b/external_plugins/voicetranser/skills/voicetranser/SKILL.md new file mode 100644 index 00000000..e7108969 --- /dev/null +++ b/external_plugins/voicetranser/skills/voicetranser/SKILL.md @@ -0,0 +1,70 @@ +--- +name: voicetranser +description: Install, configure and troubleshoot VoiceTranser — a fully local macOS push-to-talk voice input tool (hold a global hotkey, speak, release to transcribe-and-paste into any app). Use when the user wants hands-free voice dictation for coding/agents, asks to install VoiceTranser, wire its Karabiner hotkey / launchd service, switch STT engines, or fix transcription/paste/recording issues. +--- + +# VoiceTranser — local push-to-talk voice input for macOS + +Hold a global hotkey, speak, release → transcription is pasted at the cursor. +Fully local (zero external APIs). Repo: https://github.com/Lumos-789/VoiceTranser + +## Facts you need before touching anything + +- Target: macOS **Apple Silicon** (the default engine needs MLX/Metal; Intel Macs fall back to the CPU engine). +- Stack: Python 3.11+ managed by **uv**; HTTP server (stdlib); launchd service; Karabiner-Elements hotkey. +- Engines (`STT_ENGINE`): `qwen3-mlx` (default, Qwen3-ASR 0.6B 4bit via mlx-audio, RTF≈0.015, best zh quality, punctuated) → `sensevoice` (CPU fallback, RTF≈0.03) → `mlx-whisper` (en/exotic languages). +- Port: **9877** is the single default (`config.DEFAULT_SERVER_PORT`; `server.py`, `install_launchd.py` and diagnose all derive from it). `SERVER_PORT` and the installed launchd plist override it. Diagnose resolves `--port` > env > launchd plist > `.env` > default and prints a WARNING when the plist disagrees with the default. If the mic is off-limits, use `--skip-audio`. +- Model files: SenseVoice 228 MB via `download_model.py` (hf-mirror); Qwen3-ASR ~500 MB auto-fetched on first run (respects `HF_ENDPOINT`). + +## Install flow (do these in order, verify each step) + +1. **Preflight**: `uname -m` (expect arm64), `command -v uv` (if missing: `brew install uv` — ask before installing anything). +2. **Clone & deps**: + ``` + git clone https://github.com/Lumos-789/VoiceTranser.git ~/VoiceTranser && cd ~/VoiceTranser + uv sync # if PyPI crawls: UV_DEFAULT_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple uv sync + uv run python download_model.py + ``` +3. **First-run check**: `uv run python -m voicetranser --config` must print config without error. +4. **Service (recommended)**: + ``` + uv run python install_launchd.py + curl -s localhost:9877/status # expect {"status": "idle"}; model preloads in background + ``` + `install_launchd.py` writes a per-user plist whose `ProgramArguments` and + `WorkingDirectory` point at this checkout, then loads it — the repo ships no + plist snapshot, so there is nothing to copy. Use `--port`, `--engine`, or + `--no-load` when needed. +5. **Hotkey — Karabiner-Elements** complex modification (user must have Karabiner installed; add to `~/.config/karabiner/karabiner.json` → `profiles[].complex_modifications.rules`, then enable in the Karabiner UI): + ```json + { + "description": "VoiceTranser: right_option push-to-talk", + "manipulators": [ + { "type": "basic", "from": { "key_code": "right_option" }, + "to": [{ "shell_command": "curl -s localhost:9877/start > /dev/null" }], + "to_after_key_up": [{ "shell_command": "curl -s localhost:9877/stop > /dev/null" }] } + ] + } + ``` + No Karabiner? Raycast/Hammerspoon hitting `/toggle` works too. +6. **Accessibility permission** (paste step needs it, grant once): System Settings → Privacy & Security → Accessibility → enable the terminal app. Then `.venv/bin/python tests/diagnose.py` — it must return a verdict within seconds: all 8 checks pass. The audio probe is bounded (3 s) and never prompts for the microphone; without permission it reports `microphone permission not granted`. A skipped check (e.g. `--skip-audio`) counts as *not fully verified* and keeps the exit code non-zero, so it can never masquerade as green. + +## Troubleshoot (in order of likelihood) + +- **Nothing pastes** → accessibility permission missing (step 6); re-run diagnose. +- **Hotkey does nothing** → `curl -s localhost:9877/status`; if refused, `launchctl kickstart -k gui/$(id -u)/com.voicetranser-sense` and check `~/Library/Logs/VoiceTranser-sense.log`. +- **Slow first transcription after boot** → model preload still running (cold load can take ~75 s); it is one-time per service start. +- **Bad quality on specific words** → append a mapping to `corrections.json` (whole-word replace, applied post-transcription), restart service. +- **GPU/memory concerns** → engine already caps MLX buffer cache at 1 GB; to fully avoid GPU use switch: `STT_ENGINE=sensevoice` (edit plist/`.env`, restart). +- **Generic health** → `.venv/bin/python tests/diagnose.py` (8 checks: deps, models, mic, clipboard, service, logs…). Fix whatever it flags before guessing. +- **Microphone off-limits / permission prompt risk** → never let a diagnostic wait on a TCC dialog: run `.venv/bin/python tests/diagnose.py --skip-audio`. The audio check only opens a stream after a non-prompting TCC status read reports *authorized*, and the probe itself is killed after 3 s. + +## Switch engines + +Edit `STT_ENGINE` in `~/Library/LaunchAgents/com.voicetranser-sense.plist` (overrides `.env`), then `launchctl bootout gui/$(id -u)/com.voicetranser-sense && launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/com.voicetranser-sense.plist`. + +## Hard rules + +- Never send audio or transcripts anywhere — the whole point is local-only operation. +- Don't install brew packages or modify Karabiner config without asking first. +- After any config change, verify with `diagnose.py` + one real `curl /status` before declaring done. diff --git a/marketplace.json b/marketplace.json index c7f09a06..d8ab9cc0 100644 --- a/marketplace.json +++ b/marketplace.json @@ -10,7 +10,10 @@ "url": "https://z.ai" }, "unlistedPlugins": [ - { "name": "openpencil", "source": "./external_plugins/openpencil" } + { + "name": "openpencil", + "source": "./external_plugins/openpencil" + } ], "plugins": [ { @@ -827,7 +830,9 @@ }, { "name": "excalidraw", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "Excalidraw", "displayName_i18n": { "en": "Excalidraw", @@ -850,7 +855,9 @@ "name": "photocraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/photocraft/icon.png", "source": "./external_plugins/photocraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "PhotoCraft", "displayName_i18n": { "en": "PhotoCraft", @@ -877,7 +884,9 @@ "name": "vectorcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/vectorcraft/icon.png", "source": "./external_plugins/vectorcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "VectorCraft", "displayName_i18n": { "en": "VectorCraft", @@ -904,7 +913,9 @@ "name": "pdfcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/pdfcraft/icon.png", "source": "./external_plugins/pdfcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "PdfCraft", "displayName_i18n": { "en": "PdfCraft", @@ -931,7 +942,9 @@ "name": "designcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/designcraft/icon.png", "source": "./external_plugins/designcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "DesignCraft", "displayName_i18n": { "en": "DesignCraft", @@ -958,7 +971,9 @@ "name": "wordcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/wordcraft/icon.png", "source": "./external_plugins/wordcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "WordCraft", "displayName_i18n": { "en": "WordCraft", @@ -985,7 +1000,9 @@ "name": "gridcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/gridcraft/icon.png", "source": "./external_plugins/gridcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "GridCraft", "displayName_i18n": { "en": "GridCraft", @@ -1012,7 +1029,9 @@ "name": "deckcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/deckcraft/icon.png", "source": "./external_plugins/deckcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "DeckCraft", "displayName_i18n": { "en": "DeckCraft", @@ -1039,7 +1058,9 @@ "name": "cadcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/cadcraft/icon.png", "source": "./external_plugins/cadcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "CADCraft", "displayName_i18n": { "en": "CADCraft", @@ -1066,7 +1087,9 @@ "name": "soundcraft", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/soundcraft/icon.png", "source": "./external_plugins/soundcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "SoundCraft", "displayName_i18n": { "en": "SoundCraft", @@ -1149,7 +1172,9 @@ }, "category": "developer-tools", "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/kicad/icon.png", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "KiCad", "displayName_i18n": { "en": "KiCad", @@ -1443,6 +1468,42 @@ "mcp", "verification" ] + }, + { + "name": "voicetranser", + "source": "./external_plugins/voicetranser", + "icon": "https://cdn-zcode.z.ai/zcode/official-plugin/assets/voicetranser/icon.png", + "description": "Local push-to-talk voice input for macOS: hold a global hotkey, speak, release — transcription is pasted at your cursor. Fully local STT (Qwen3-ASR MLX, SenseVoice, mlx-whisper), zero external APIs. The bundled skill guides your agent through install, hotkey wiring, launchd setup and troubleshooting.", + "description_i18n": { + "en": "Local push-to-talk voice input for macOS: hold a global hotkey, speak, release — transcription is pasted at your cursor. Fully local STT (Qwen3-ASR MLX, SenseVoice, mlx-whisper), zero external APIs. The bundled skill guides your agent through install, hotkey wiring, launchd setup and troubleshooting.", + "zh-CN": "macOS 本地按住说话语音输入：按住全局热键说话、松开即转写粘贴到光标处。纯本地 STT（Qwen3-ASR MLX / SenseVoice / mlx-whisper），零外部 API。内置技能引导 agent 完成安装、热键接线、launchd 常驻与排障。" + }, + "version": "1.0.3", + "author": { + "name": "Lumos-789", + "url": "https://github.com/Lumos-789" + }, + "category": "productivity", + "keywords": [ + "voice-input", + "speech-to-text", + "dictation", + "macos", + "push-to-talk", + "local-first", + "qwen3-asr", + "sensevoice" + ], + "examplePrompts": [ + "Install VoiceTranser and wire my right Option key for push-to-talk", + "语音输入装一下，右 Option 按住说话" + ], + "examplePrompts_i18n": { + "zh-CN": [ + "语音输入装一下，右 Option 按住说话", + "转写没粘贴出来，帮我排查 VoiceTranser" + ] + } } ] -} \ No newline at end of file +}