Phase 2: On-device Whisper — no API, no internet, no cost #4000
4 changed files+307−7
Modifiedtauri-app/src-tauri/Cargo.toml+2−0View fileUnifiedSplit
@@ -35,3 +35,5 @@ cpal = "0.15" # Cross-platform audio capture
3535hound = "3.5" # WAV encoding for Whisper API
3636regex = "1" # Text processing patterns
3737once_cell = "1" # Lazy static initialization
38whisper-rs = "0.12" # whisper.cpp Rust bindings — on-device speech-to-text
39dirs = "5" # OS-standard directories for model storage
Modifiedtauri-app/src-tauri/src/lib.rs+70−7View fileUnifiedSplit
@@ -14,6 +14,7 @@ mod audio;
1414mod keyboard;
1515mod transcribe;
1616mod grammar;
17mod local_whisper;
1718
1819use std::sync::{Arc, Mutex};
1920use tauri::{
@@ -28,6 +29,8 @@ pub struct AppState {
2829 pub claude_api_key: Mutex<String>,
2930 pub language: Mutex<String>,
3031 pub ai_rewrite: Mutex<bool>,
32 pub use_local_whisper: Mutex<bool>, // true = on-device, false = API
33 pub local_model: Mutex<String>, // model filename e.g. "ggml-base.bin"
3134}
3235
3336impl Default for AppState {
@@ -38,6 +41,8 @@ impl Default for AppState {
3841 claude_api_key: Mutex::new(String::new()),
3942 language: Mutex::new("en".to_string()),
4043 ai_rewrite: Mutex::new(false),
44 use_local_whisper: Mutex::new(false),
45 local_model: Mutex::new("ggml-base.bin".to_string()),
4146 }
4247 }
4348}
@@ -55,17 +60,44 @@ async fn toggle_recording(state: tauri::State<'_, Arc<AppState>>, app: AppHandle
5560 // Get recorded audio and transcribe
5661 let audio_data = audio::stop_recording();
5762
58 let api_key = state.whisper_api_key.lock().unwrap().clone();
63 let use_local = *state.use_local_whisper.lock().unwrap();
5964 let language = state.language.lock().unwrap().clone();
6065
61 if api_key.is_empty() {
62 return Err("No API key set. Open Settings to add your OpenAI key.".to_string());
63 }
66 let text = if use_local {
67 // On-device transcription — no API key, no internet, no cost
68 let model_name = state.local_model.lock().unwrap().clone();
69 let model_file = local_whisper::model_path(&model_name);
70
71 if !model_file.exists() {
72 return Err(format!(
73 "Model '{}' not downloaded yet. Go to Settings → Download Model.",
74 model_name
75 ));
76 }
77
78 let model_str = model_file.to_string_lossy().to_string();
79 let audio = audio_data.clone();
80 let lang = language.clone();
6481
65 // Transcribe with Whisper
66 let text = transcribe::whisper_api(&audio_data, &api_key, &language)
82 // Run in blocking thread (Whisper inference is CPU-bound)
83 tokio::task::spawn_blocking(move || {
84 local_whisper::transcribe_local(&audio, &model_str, &lang)
85 })
6786 .await
68 .map_err(|e| format!("Transcription failed: {}", e))?;
87 .map_err(|e| format!("Thread error: {}", e))?
88 .map_err(|e| format!("Local transcription failed: {}", e))?
89 } else {
90 // Cloud transcription via Whisper API
91 let api_key = state.whisper_api_key.lock().unwrap().clone();
92
93 if api_key.is_empty() {
94 return Err("No API key set. Open Settings to add your OpenAI key, or enable Local Whisper.".to_string());
95 }
96
97 transcribe::whisper_api(&audio_data, &api_key, &language)
98 .await
99 .map_err(|e| format!("Transcription failed: {}", e))?
100 };
69101
70102 if text.is_empty() {
71103 return Ok("No speech detected.".to_string());
@@ -130,6 +162,32 @@ fn set_ai_rewrite(state: tauri::State<'_, Arc<AppState>>, enabled: bool) {
130162 *state.ai_rewrite.lock().unwrap() = enabled;
131163}
132164
165
166fn set_use_local_whisper(state: tauri::State<'_, Arc<AppState>>, enabled: bool) {
167 *state.use_local_whisper.lock().unwrap() = enabled;
168}
169
170
171fn set_local_model(state: tauri::State<'_, Arc<AppState>>, model: String) {
172 *state.local_model.lock().unwrap() = model;
173}
174
175
176fn check_model_downloaded(model_name: String) -> bool {
177 local_whisper::model_exists(&model_name)
178}
179
180
181async fn download_model(model_name: String) -> Result<String, String> {
182 let path = local_whisper::download_model(&model_name).await?;
183 Ok(format!("Model downloaded: {:?}", path))
184}
185
186
187fn get_models_dir() -> String {
188 local_whisper::models_dir().to_string_lossy().to_string()
189}
190
133191fn update_tray(app: &AppHandle, recording: bool) {
134192 if let Some(tray) = app.tray_by_id("main") {
135193 let _ = tray.set_tooltip(Some(if recording {
@@ -230,6 +288,11 @@ pub fn run() {
230288 set_claude_key,
231289 set_language,
232290 set_ai_rewrite,
291 set_use_local_whisper,
292 set_local_model,
293 check_model_downloaded,
294 download_model,
295 get_models_dir,
233296 ])
234297 .run(tauri::generate_context!())
235298 .expect("error while running 48co");
Addedtauri-app/src-tauri/src/local_whisper.rs+163−0View fileUnifiedSplit
@@ -0,0 +1,163 @@
1// Local on-device transcription using whisper.cpp (via whisper-rs)
2//
3// No API key needed. No internet needed. No cost per use.
4// Runs the Whisper model directly on the user's CPU/GPU.
5//
6// Model files are downloaded once and stored in the app data directory:
7// Windows: %APPDATA%/48co/models/
8// macOS: ~/Library/Application Support/48co/models/
9// Linux: ~/.local/share/48co/models/
10
11use std::path::PathBuf;
12
13/// Get the models directory
14pub fn models_dir() -> PathBuf {
15 let base = dirs::data_dir()
16 .unwrap_or_else(|| PathBuf::from("."))
17 .join("48co")
18 .join("models");
19 std::fs::create_dir_all(&base).ok();
20 base
21}
22
23/// Check if a model is downloaded
24pub fn model_exists(model_name: &str) -> bool {
25 models_dir().join(model_name).exists()
26}
27
28/// Get the path to a model file
29pub fn model_path(model_name: &str) -> PathBuf {
30 models_dir().join(model_name)
31}
32
33/// Download a Whisper model from Hugging Face
34/// Models available:
35/// - ggml-tiny.bin (~75MB, fastest, least accurate)
36/// - ggml-base.bin (~142MB, good balance)
37/// - ggml-small.bin (~466MB, good accuracy)
38/// - ggml-medium.bin (~1.5GB, very good accuracy)
39/// - ggml-large-v3.bin (~3GB, best accuracy)
40/// - ggml-large-v3-turbo.bin (~1.6GB, fast + accurate — RECOMMENDED)
41pub async fn download_model(model_name: &str) -> Result<PathBuf, String> {
42 let path = model_path(model_name);
43 if path.exists() {
44 return Ok(path);
45 }
46
47 let url = format!(
48 "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/{}",
49 model_name
50 );
51
52 println!("[48co] Downloading model: {} ...", model_name);
53
54 let client = reqwest::Client::new();
55 let response = client
56 .get(&url)
57 .send()
58 .await
59 .map_err(|e| format!("Download failed: {}", e))?;
60
61 if !response.status().is_success() {
62 return Err(format!("Download failed: HTTP {}", response.status()));
63 }
64
65 let bytes = response
66 .bytes()
67 .await
68 .map_err(|e| format!("Download read failed: {}", e))?;
69
70 std::fs::write(&path, &bytes)
71 .map_err(|e| format!("Save failed: {}", e))?;
72
73 println!("[48co] Model saved: {:?} ({:.1}MB)", path, bytes.len() as f64 / 1_048_576.0);
74
75 Ok(path)
76}
77
78/// Transcribe audio using local Whisper model
79/// Takes WAV audio data (16kHz mono) and returns text
80pub fn transcribe_local(audio_wav: &[u8], model_path: &str, language: &str) -> Result<String, String> {
81 use whisper_rs::{WhisperContext, WhisperContextParameters, FullParams, SamplingStrategy};
82
83 if audio_wav.is_empty() {
84 return Ok(String::new());
85 }
86
87 // Load the model
88 let ctx = WhisperContext::new_with_params(
89 model_path,
90 WhisperContextParameters::default(),
91 ).map_err(|e| format!("Model load failed: {}", e))?;
92
93 // Decode WAV to f32 samples
94 let samples = decode_wav_to_f32(audio_wav)?;
95
96 if samples.is_empty() {
97 return Ok(String::new());
98 }
99
100 // Set up transcription parameters
101 let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
102 params.set_language(Some(language));
103 params.set_print_special(false);
104 params.set_print_progress(false);
105 params.set_print_realtime(false);
106 params.set_print_timestamps(false);
107 params.set_suppress_blank(true);
108 params.set_single_segment(false);
109
110 // Run transcription
111 let mut state = ctx.create_state()
112 .map_err(|e| format!("State creation failed: {}", e))?;
113
114 state.full(params, &samples)
115 .map_err(|e| format!("Transcription failed: {}", e))?;
116
117 // Collect results
118 let num_segments = state.full_n_segments()
119 .map_err(|e| format!("Segment count failed: {}", e))?;
120
121 let mut text = String::new();
122 for i in 0..num_segments {
123 if let Ok(segment) = state.full_get_segment_text(i) {
124 text.push_str(&segment);
125 }
126 }
127
128 Ok(text.trim().to_string())
129}
130
131/// Decode WAV bytes to f32 samples (16kHz mono expected by Whisper)
132fn decode_wav_to_f32(wav_data: &[u8]) -> Result<Vec<f32>, String> {
133 let cursor = std::io::Cursor::new(wav_data);
134 let reader = hound::WavReader::new(cursor)
135 .map_err(|e| format!("WAV decode failed: {}", e))?;
136
137 let spec = reader.spec();
138 let samples: Vec<f32> = match spec.sample_format {
139 hound::SampleFormat::Int => {
140 let max = (1 << (spec.bits_per_sample - 1)) as f32;
141 reader.into_samples::<i32>()
142 .filter_map(|s| s.ok())
143 .map(|s| s as f32 / max)
144 .collect()
145 }
146 hound::SampleFormat::Float => {
147 reader.into_samples::<f32>()
148 .filter_map(|s| s.ok())
149 .collect()
150 }
151 };
152
153 // If stereo, convert to mono by averaging channels
154 if spec.channels == 2 {
155 let mono: Vec<f32> = samples
156 .chunks(2)
157 .map(|chunk| (chunk[0] + chunk.get(1).copied().unwrap_or(0.0)) / 2.0)
158 .collect();
159 Ok(mono)
160 } else {
161 Ok(samples)
162 }
163}
Modifiedtauri-app/src/App.jsx+72−0View fileUnifiedSplit
@@ -7,6 +7,10 @@ export default function App() {
77 const [claudeKey, setClaudeKey] = useState('')
88 const [language, setLanguage] = useState('en')
99 const [aiRewrite, setAiRewrite] = useState(false)
10 const [useLocalWhisper, setUseLocalWhisper] = useState(false)
11 const [localModel, setLocalModel] = useState('ggml-base.bin')
12 const [modelDownloaded, setModelDownloaded] = useState(false)
13 const [downloading, setDownloading] = useState(false)
1014 const [status, setStatus] = useState('Ready')
1115 const [saved, setSaved] = useState(false)
1216
@@ -19,12 +23,21 @@ export default function App() {
1923 setClaudeKey(await store.get('claudeApiKey') || '')
2024 setLanguage(await store.get('language') || 'en')
2125 setAiRewrite(await store.get('aiRewrite') || false)
26 setUseLocalWhisper(await store.get('useLocalWhisper') || false)
27 setLocalModel(await store.get('localModel') || 'ggml-base.bin')
2228
2329 // Send to Rust backend
2430 if (await store.get('whisperApiKey')) invoke('set_api_key', { key: await store.get('whisperApiKey') })
2531 if (await store.get('claudeApiKey')) invoke('set_claude_key', { key: await store.get('claudeApiKey') })
2632 if (await store.get('language')) invoke('set_language', { lang: await store.get('language') })
2733 invoke('set_ai_rewrite', { enabled: await store.get('aiRewrite') || false })
34 invoke('set_use_local_whisper', { enabled: await store.get('useLocalWhisper') || false })
35 invoke('set_local_model', { model: await store.get('localModel') || 'ggml-base.bin' })
36
37 // Check if model is downloaded
38 const model = await store.get('localModel') || 'ggml-base.bin'
39 const downloaded = await invoke('check_model_downloaded', { modelName: model })
40 setModelDownloaded(downloaded)
2841 } catch (e) {
2942 console.warn('Settings load failed:', e)
3043 }
@@ -39,6 +52,8 @@ export default function App() {
3952 await store.set('claudeApiKey', claudeKey)
4053 await store.set('language', language)
4154 await store.set('aiRewrite', aiRewrite)
55 await store.set('useLocalWhisper', useLocalWhisper)
56 await store.set('localModel', localModel)
4257 await store.save()
4358
4459 // Update Rust backend
@@ -46,6 +61,8 @@ export default function App() {
4661 await invoke('set_claude_key', { key: claudeKey })
4762 await invoke('set_language', { lang: language })
4863 await invoke('set_ai_rewrite', { enabled: aiRewrite })
64 await invoke('set_use_local_whisper', { enabled: useLocalWhisper })
65 await invoke('set_local_model', { model: localModel })
4966
5067 setSaved(true)
5168 setTimeout(() => setSaved(false), 2000)
@@ -97,6 +114,61 @@ export default function App() {
97114 <p className="text-xs text-gray-400 mt-1">For AI grammar rewrite. ~$0.003/rewrite</p>
98115 </div>
99116
117 {/* Local Whisper */}
118 <div className="mb-6 p-4 rounded-lg bg-gray-50 border border-gray-100">
119 <div className="flex items-center justify-between mb-3">
120 <div>
121 <p className="text-sm font-medium text-gray-700">Local Voice (no API needed)</p>
122 <p className="text-xs text-gray-400">Runs Whisper on your device. Free, private, works offline.</p>
123 </div>
124 <button
125 onClick={() => setUseLocalWhisper(!useLocalWhisper)}
126 className={`w-10 h-6 rounded-full transition-colors ${useLocalWhisper ? 'bg-indigo-600' : 'bg-gray-200'}`}
127 >
128 <div className={`w-4 h-4 bg-white rounded-full shadow transform transition-transform mx-1 ${useLocalWhisper ? 'translate-x-4' : ''}`} />
129 </button>
130 </div>
131
132 {useLocalWhisper && (
133 <>
134 <select
135 value={localModel}
136 onChange={(e) => { setLocalModel(e.target.value); setModelDownloaded(false) }}
137 className="w-full px-3 py-2 border border-gray-200 rounded-lg text-sm mb-2 focus:outline-none focus:border-indigo-300"
138 >
139 <option value="ggml-tiny.bin">Tiny (~75MB, fastest)</option>
140 <option value="ggml-base.bin">Base (~142MB, recommended)</option>
141 <option value="ggml-small.bin">Small (~466MB, more accurate)</option>
142 <option value="ggml-medium.bin">Medium (~1.5GB, very accurate)</option>
143 <option value="ggml-large-v3-turbo.bin">Large V3 Turbo (~1.6GB, best)</option>
144 </select>
145
146 {modelDownloaded ? (
147 <p className="text-xs text-green-600">Model ready</p>
148 ) : (
149 <button
150 onClick={async () => {
151 setDownloading(true)
152 setStatus('Downloading model...')
153 try {
154 await invoke('download_model', { modelName: localModel })
155 setModelDownloaded(true)
156 setStatus('Model downloaded!')
157 } catch (e) {
158 setStatus('Download failed: ' + e)
159 }
160 setDownloading(false)
161 }}
162 disabled={downloading}
163 className="w-full py-2 rounded-lg bg-indigo-100 text-indigo-700 text-xs font-medium hover:bg-indigo-200 transition-colors disabled:opacity-50"
164 >
165 {downloading ? 'Downloading...' : `Download ${localModel}`}
166 </button>
167 )}
168 </>
169 )}
170 </div>
171
100172 {/* Language */}
101173 <div className="mb-6">
102174 <label className="block text-sm font-medium text-gray-700 mb-1">Language</label>
103175
c comment · e edit title · m merge · a approve · r request changes · ? shortcuts