From e57d86c26c0fe5ae90d4f7ab3714917e38b5d904 Mon Sep 17 00:00:00 2001 From: Xavier Lau Date: Tue, 9 Jul 2024 04:19:54 +0800 Subject: [PATCH] Optimize keyboard events --- Cargo.lock | 2 - Cargo.toml | 5 +- src/component.rs | 19 +++++- src/component/keyboard.rs | 109 +++++++++++++++++++++++++++--- src/component/setting.rs | 26 ++++---- src/error.rs | 14 +++- src/main.rs | 2 +- src/service.rs | 13 +--- src/service/chat.rs | 58 ++++++++-------- src/service/hotkey.rs | 137 +++++++++++++++++++++----------------- src/service/keyboard.rs | 9 ++- src/ui/panel/chat.rs | 4 ++ src/ui/panel/setting.rs | 10 +-- 13 files changed, 267 insertions(+), 141 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f510d3a..daabf3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -155,7 +155,6 @@ dependencies = [ "tracing", "tracing-appender", "tracing-subscriber", - "xkeysym", ] [[package]] @@ -1755,7 +1754,6 @@ dependencies = [ "keyboard-types", "objc", "once_cell", - "serde", "thiserror", "windows-sys 0.52.0", "x11-dl", diff --git a/Cargo.toml b/Cargo.toml index 5c59774..0461683 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ eframe = { version = "0.28", features = ["persistence"] } egui_extras = { version = "0.28", features = ["svg"] } enigo = { version = "0.2" } futures = { version = "0.3" } -global-hotkey = { version = "0.5", features = ["serde"] } +global-hotkey = { version = "0.5" } parking_lot = { version = "0.12" } reqwew = { version = "0.2" } serde = { version = "1.0", features = ["derive"] } @@ -49,9 +49,6 @@ tracing-appender = { version = "0.2" } tracing-subscriber = { version = "0.3", features = ["env-filter"] } # llm_utils = { version = "0.0.6", optional = true } -[target.'cfg(all(unix, not(target_os = "macos")))'.dependencies] -xkeysym = { version = "0.2" } - [target.'cfg(target_os = "macos")'.dependencies] objc2-app-kit = { version = "0.2", features = ["NSApplication", "NSResponder", "NSRunningApplication", "NSWindow"] } objc2-foundation = { version = "0.2" } diff --git a/src/component.rs b/src/component.rs index 63f37cb..75f9108 100644 --- a/src/component.rs +++ b/src/component.rs @@ -16,17 +16,22 @@ use setting::Setting; pub mod util; +// std +use std::fmt::{Debug, Formatter, Result as FmtResult}; +// crates.io +use arboard::Clipboard; // self use crate::prelude::*; -#[derive(Debug)] pub struct Components { + pub clipboard: Clipboard, pub setting: Setting, #[cfg(feature = "tokenizer")] pub tokenizer: Tokenizer, } impl Components { pub fn new() -> Result { + let clipboard = Clipboard::new()?; let setting = Setting::load()?; // TODO: https://github.com/emilk/egui/discussions/4670. @@ -36,9 +41,21 @@ impl Components { let tokenizer = Tokenizer::new(setting.ai.model.as_str()); Ok(Self { + clipboard, setting, #[cfg(feature = "tokenizer")] tokenizer, }) } } +impl Debug for Components { + fn fmt(&self, f: &mut Formatter) -> FmtResult { + let mut s = f.debug_struct("Components"); + + s.field("clipboard", &"..").field("setting", &self.setting); + #[cfg(feature = "tokenizer")] + s.field("tokenizer", &self.tokenizer); + + s.finish() + } +} diff --git a/src/component/keyboard.rs b/src/component/keyboard.rs index 9d41719..9e0e3d9 100644 --- a/src/component/keyboard.rs +++ b/src/component/keyboard.rs @@ -1,6 +1,7 @@ +// std +use std::str::FromStr; // crates.io use enigo::{Direction, Enigo, Key, Keyboard as _, Settings}; -#[cfg(all(unix, not(target_os = "macos")))] use xkeysym::key::c; // self use crate::prelude::*; @@ -12,14 +13,19 @@ impl Keyboard { } pub fn copy(&mut self) -> Result<()> { - self.0.key(Key::Meta, Direction::Press).map_err(EnigoError::Input)?; - // TODO: create a `CGKeyCode` table for macOS in `build.rs`. - #[cfg(target_os = "macos")] - self.0.key(Key::Other(0x08), Direction::Click).map_err(EnigoError::Input)?; - // TODO: Windows. - #[cfg(all(unix, not(target_os = "macos")))] - self.0.key(Key::Other(c), Direction::Click).map_err(EnigoError::Input)?; - self.0.key(Key::Meta, Direction::Release).map_err(EnigoError::Input)?; + let modifier = if cfg!(target_os = "macos") { Key::Meta } else { Key::Control }; + + self.0.key(modifier, Direction::Press).map_err(EnigoError::Input)?; + self.0.key(key_of('C')?, Direction::Click).map_err(EnigoError::Input)?; + self.0.key(modifier, Direction::Release).map_err(EnigoError::Input)?; + + Ok(()) + } + + pub fn release_keys(&mut self, keys: Keys) -> Result<()> { + for k in keys.0 { + self.0.key(k, Direction::Release).map_err(EnigoError::Input)?; + } Ok(()) } @@ -28,3 +34,88 @@ impl Keyboard { Ok(self.0.text(text).map_err(EnigoError::Input)?) } } + +#[derive(Clone, Debug)] +pub struct Keys(pub Vec); +impl FromStr for Keys { + type Err = Error; + + fn from_str(s: &str) -> Result { + let mut keys = Vec::new(); + + for k in s.to_uppercase().split('+') { + let k = match k { + "CTRL" | "CONTROL" => Key::Control, + "SHIFT" => Key::Shift, + "ALT" => Key::Alt, + "COMMAND" | "META" | "SUPER" => Key::Meta, + k if k.len() == 1 => key_of(k.chars().next().expect("`k` must be `char`"))?, + k => return Err(Error::UnsupportedKey(k.to_owned())), + }; + + keys.push(k); + } + + Ok(Self(keys)) + } +} + +fn key_of(key: char) -> Result { + // TODO: create a `CGKeyCode` table for macOS in `build.rs`. + // Currently, we only support limited keys on macOS. + #[cfg(target_os = "macos")] + let k = Key::Other(match key { + 'A' => 0, + 'S' => 1, + 'D' => 2, + 'F' => 3, + 'H' => 4, + 'G' => 5, + 'Z' => 6, + 'X' => 7, + 'C' => 8, + 'V' => 9, + 'B' => 11, + 'Q' => 12, + 'W' => 13, + 'E' => 14, + 'R' => 15, + 'Y' => 16, + 'T' => 17, + '1' => 18, + '2' => 19, + '3' => 20, + '4' => 21, + '6' => 22, + '5' => 23, + '=' => 24, + '9' => 25, + '7' => 26, + '-' => 27, + '8' => 28, + '0' => 29, + ']' => 30, + 'O' => 31, + 'U' => 32, + '[' => 33, + 'I' => 34, + 'P' => 35, + 'L' => 37, + 'J' => 38, + '\'' => 39, + 'K' => 40, + ';' => 41, + '\\' => 42, + ',' => 43, + '/' => 44, + 'N' => 45, + 'M' => 46, + '.' => 47, + '`' => 50, + _ => return Err(Error::UnsupportedKey(key.to_string())), + }); + #[cfg(not(target_os = "macos"))] + let k = Key::Unicode(key); + + Ok(k) +} diff --git a/src/component/setting.rs b/src/component/setting.rs index a64df21..bcb584f 100644 --- a/src/component/setting.rs +++ b/src/component/setting.rs @@ -3,7 +3,6 @@ use std::{borrow::Cow, fs, path::PathBuf}; // crates.io use app_dirs2::AppDataType; use async_openai::config::OPENAI_API_BASE; -use global_hotkey::hotkey::{Code, HotKey, Modifiers}; use serde::{Deserialize, Serialize}; // self use super::openai::Model; @@ -101,7 +100,8 @@ impl Default for Rewrite { fn default() -> Self { Self { prompt: "As language professor, assist me in refining this text. \ - Amend any grammatical errors and enhance the language to sound more like a native speaker.\ + Amend any grammatical errors, \ + enhance the language to sound more like a native speaker and keep the origin format. \ Just provide the refined text only, without any other things:" .into(), } @@ -127,8 +127,10 @@ impl Translation { impl Default for Translation { fn default() -> Self { Self { - prompt: "As a language professor, amend any grammatical errors and enhance the language to sound more like a native speaker. \ - Provide the translated text only, without any other things:".into(), + prompt: "As a language professor, amend any grammatical errors, \ + enhance the language to sound more like a native speaker and keep the origin format. \ + Provide the translated text only, without any other things:" + .into(), a: Language::ZhCn, b: Language::EnGb, } @@ -147,18 +149,18 @@ pub enum Language { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub struct Hotkeys { - pub rewrite: HotKey, - pub rewrite_directly: HotKey, - pub translate: HotKey, - pub translate_directly: HotKey, + pub rewrite: String, + pub rewrite_directly: String, + pub translate: String, + pub translate_directly: String, } impl Default for Hotkeys { fn default() -> Self { Self { - rewrite: HotKey::new(Some(Modifiers::CONTROL), Code::KeyT), - rewrite_directly: HotKey::new(Some(Modifiers::CONTROL), Code::KeyY), - translate: HotKey::new(Some(Modifiers::CONTROL), Code::KeyU), - translate_directly: HotKey::new(Some(Modifiers::CONTROL), Code::KeyI), + rewrite: "Ctrl+Y".into(), + rewrite_directly: "Ctrl+U".into(), + translate: "Ctrl+I".into(), + translate_directly: "Ctrl+O".into(), } } } diff --git a/src/error.rs b/src/error.rs index 635fe37..a15926d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -12,8 +12,6 @@ pub enum Error { #[error(transparent)] Eframe(#[from] eframe::Error), #[error(transparent)] - GlobalHotKey(#[from] global_hotkey::Error), - #[error(transparent)] OpenAi(#[from] async_openai::error::OpenAIError), #[error(transparent)] Reqwew(#[from] reqwew::error::Error), @@ -22,6 +20,10 @@ pub enum Error { #[error(transparent)] Enigo(#[from] EnigoError), + #[error(transparent)] + GlobalHotKey(#[from] GlobalHotKeyError), + #[error("unsupported key: {0}")] + UnsupportedKey(String) } #[derive(Debug, thiserror::Error)] @@ -31,3 +33,11 @@ pub enum EnigoError { #[error(transparent)] NewCon(#[from] enigo::NewConError), } + +#[derive(Debug, thiserror::Error)] +pub enum GlobalHotKeyError { + #[error(transparent)] + Main(#[from] global_hotkey::Error), + #[error(transparent)] + Parse(#[from] global_hotkey::hotkey::HotKeyParseError), +} diff --git a/src/main.rs b/src/main.rs index 38d2c93..bc54738 100644 --- a/src/main.rs +++ b/src/main.rs @@ -18,7 +18,7 @@ mod state; mod ui; mod prelude { - pub type Result = std::result::Result; + pub type Result = std::result::Result; pub use crate::error::*; } diff --git a/src/service.rs b/src/service.rs index 59eeac2..f7ebfa6 100644 --- a/src/service.rs +++ b/src/service.rs @@ -36,17 +36,10 @@ impl Services { let rt = Runtime::new()?; let quoter = Quoter::new(&rt, state.chat.quote.clone()); let is_chatting = Arc::new(AtomicBool::new(false)); - let chat = Chat::new( - keyboard.clone(), - &rt, - is_chatting.clone(), - components.setting.ai.clone(), - components.setting.chat.clone(), - state.chat.input.clone(), - state.chat.output.clone(), - ); + let chat = + Chat::new(keyboard.clone(), &rt, is_chatting.clone(), &components.setting, &state.chat); let hotkey = - Hotkey::new(ctx, keyboard.clone(), &rt, &components.setting.hotkeys, chat.tx.clone())?; + Hotkey::new(ctx, keyboard.clone(), &components.setting.hotkeys, chat.tx.clone())?; Ok(Self { keyboard, rt: Some(rt), quoter, is_chatting, chat, hotkey }) } diff --git a/src/service/chat.rs b/src/service/chat.rs index e23f1ad..5744545 100644 --- a/src/service/chat.rs +++ b/src/service/chat.rs @@ -9,14 +9,16 @@ use std::{ }; // crates.io use futures::StreamExt; -use parking_lot::RwLock; -use tokio::{runtime::Runtime, task::AbortHandle, time}; +use tokio::{runtime::Runtime, sync::Mutex, task::AbortHandle, time}; // self use super::keyboard::Keyboard; -use crate::component::{ - function::Function, - openai::OpenAi, - setting::{Ai, Chat as ChatSetting}, +use crate::{ + component::{ + function::Function, + openai::OpenAi, + setting::{Chat as ChatSetting, Setting}, + }, + state::Chat as ChatState, }; pub type ChatArgs = (Function, String, bool); @@ -24,6 +26,9 @@ pub type ChatArgs = (Function, String, bool); #[derive(Debug)] pub struct Chat { pub tx: Sender, + // TODO?: get rid of the `Mutex`. + openai: Arc>, + chat_setting: Arc>, abort_handle: AbortHandle, } impl Chat { @@ -31,12 +36,15 @@ impl Chat { keyboard: Keyboard, rt: &Runtime, is_chatting: Arc, - ai_setting: Ai, - chat_setting: ChatSetting, - input: Arc>, - output: Arc>, + setting: &Setting, + state: &ChatState, ) -> Self { - let openai = OpenAi::new(ai_setting); + let openai = Arc::new(Mutex::new(OpenAi::new(setting.ai.clone()))); + let openai_ = openai.clone(); + let chat_setting = Arc::new(Mutex::new(setting.chat.clone())); + let chat_setting_ = chat_setting.clone(); + let input = state.input.clone(); + let output = state.output.clone(); let (tx, rx) = mpsc::channel(); // TODO: handle the error. let abort_handle = rt @@ -52,8 +60,12 @@ impl Chat { input.write().clone_from(&content); output.write().clear(); - let mut stream = - openai.chat(&func.prompt(&chat_setting), &content).await.unwrap(); + let mut stream = openai_ + .lock() + .await + .chat(&func.prompt(&*chat_setting_.lock().await), &content) + .await + .unwrap(); while let Some(r) = stream.next().await { for s in r.unwrap().choices.into_iter().filter_map(|c| c.delta.content) { @@ -74,27 +86,15 @@ impl Chat { }) .abort_handle(); - Self { abort_handle, tx } + Self { tx, openai, chat_setting, abort_handle } } pub fn abort(&self) { self.abort_handle.abort(); } - // TODO: fix clippy. - #[allow(clippy::too_many_arguments)] - pub fn renew( - &mut self, - keyboard: Keyboard, - rt: &Runtime, - is_chatting: Arc, - ai_setting: Ai, - chat_setting: ChatSetting, - input: Arc>, - output: Arc>, - ) { - self.abort(); - - *self = Self::new(keyboard, rt, is_chatting, ai_setting, chat_setting, input, output); + pub fn renew(&mut self, setting: &Setting) { + *self.openai.blocking_lock() = OpenAi::new(setting.ai.clone()); + *self.chat_setting.blocking_lock() = setting.chat.clone(); } } diff --git a/src/service/hotkey.rs b/src/service/hotkey.rs index 14af19d..849a01f 100644 --- a/src/service/hotkey.rs +++ b/src/service/hotkey.rs @@ -1,85 +1,87 @@ // std -use std::{sync::mpsc::Sender, time::Duration}; +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + mpsc::Sender, + Arc, + }, + thread, + time::Duration, +}; // crates.io use arboard::Clipboard; use eframe::egui::{Context, ViewportCommand}; use global_hotkey::{GlobalHotKeyEvent, GlobalHotKeyManager, HotKeyState}; -use tokio::{runtime::Runtime, task::AbortHandle, time}; // self use super::{chat::ChatArgs, keyboard::Keyboard}; use crate::{ - component::{function::Function, setting::Hotkeys}, + component::{function::Function, keyboard::Keys, setting::Hotkeys}, os::*, prelude::*, }; #[derive(Debug)] -pub struct Hotkey(AbortHandle); +pub struct Hotkey(Arc); impl Hotkey { pub fn new( ctx: &Context, keyboard: Keyboard, - rt: &Runtime, hotkeys: &Hotkeys, tx: Sender, ) -> Result { let ctx = ctx.to_owned(); let manager = Manager::new(hotkeys)?; - let receiver = GlobalHotKeyEvent::receiver(); + let abort = Arc::new(AtomicBool::new(false)); + let abort_ = abort.clone(); + let hk_rx = GlobalHotKeyEvent::receiver(); let mut clipboard = Clipboard::new()?; - // TODO: handle the error. - let abort_handle = rt - .spawn(async move { - // The manager need to be kept alive during the whole program life. - let manager = manager; - loop { - // Block the thread until a hotkey event is received. - let e = receiver.recv().unwrap(); - - // We don't care about the release event. - if let HotKeyState::Pressed = e.state { - // TODO: reset the hotkey state so that we don't need to wait for the user - // to release the keys. + // TODO: handle the error. + thread::spawn(move || { + // The manager need to be kept alive during the whole program life. + let manager = manager; - let func = manager.match_func(e.id); - let to_unhide = !func.is_directly(); + while !abort_.load(Ordering::Relaxed) { + // Block the thread until a hotkey event is received. + let e = hk_rx.recv().unwrap(); - if to_unhide { - Os::unhide(); - } + // We don't care about the release event. + if let HotKeyState::Pressed = e.state { + let (func, keys) = manager.match_func(e.id); + let to_unhide = !func.is_directly(); - // Sleep for a while to reset the keyboard state after user - // triggers the hotkey. - time::sleep(Duration::from_millis(1000)).await; + if to_unhide { + Os::unhide(); + } - keyboard.copy(); + // Reset the keys' state after the user triggers them. + // If the user is still holding the keys, we can still perform the copy + // operation successfully. + keyboard.release_keys(keys); + keyboard.copy(); - // Give some time to the system to refresh the clipboard. - time::sleep(Duration::from_millis(500)).await; + // Give some time to the system to refresh the clipboard. + thread::sleep(Duration::from_millis(500)); - let content = match clipboard.get_text() { - Ok(c) if !c.is_empty() => c, - _ => continue, - }; + let content = match clipboard.get_text() { + Ok(c) if !c.is_empty() => c, + _ => continue, + }; - tx.send((func, content, !to_unhide)).unwrap(); + tx.send((func, content, !to_unhide)).unwrap(); - if to_unhide { - // Generally, this needs some time to wait the window available - // first, but the previous sleep in get selected text is enough. - ctx.send_viewport_cmd(ViewportCommand::Focus); - } + if to_unhide { + ctx.send_viewport_cmd(ViewportCommand::Focus); } } - }) - .abort_handle(); + } + }); - Ok(Self(abort_handle)) + Ok(Self(abort)) } pub fn abort(&self) { - self.0.abort(); + self.0.store(true, Ordering::Release); } // TODO: fn renew. @@ -89,30 +91,43 @@ struct Manager { // The manager need to be kept alive during the whole program life. _inner: GlobalHotKeyManager, ids: [u32; 4], + hotkeys_keys: [Keys; 4], } impl Manager { fn new(hotkeys: &Hotkeys) -> Result { - let _inner = GlobalHotKeyManager::new()?; - let hotkeys = [ - hotkeys.rewrite, - hotkeys.rewrite_directly, - hotkeys.translate, - hotkeys.translate_directly, + let _inner = GlobalHotKeyManager::new().map_err(GlobalHotKeyError::Main)?; + let hotkeys_raw = [ + &hotkeys.rewrite, + &hotkeys.rewrite_directly, + &hotkeys.translate, + &hotkeys.translate_directly, ]; - - _inner.register_all(&hotkeys)?; - - let ids = hotkeys.iter().map(|h| h.id).collect::>().try_into().unwrap(); - - Ok(Self { _inner, ids }) + let hotkeys = hotkeys_raw + .iter() + .map(|h| h.parse()) + .collect::, _>>() + .map_err(GlobalHotKeyError::Parse)?; + + _inner.register_all(&hotkeys).map_err(GlobalHotKeyError::Main)?; + + let ids = + hotkeys.iter().map(|h| h.id).collect::>().try_into().expect("array must fit"); + let hotkeys_keys = hotkeys_raw + .iter() + .map(|h| h.parse()) + .collect::, _>>()? + .try_into() + .expect("array must fit"); + + Ok(Self { _inner, ids, hotkeys_keys }) } - fn match_func(&self, id: u32) -> Function { + fn match_func(&self, id: u32) -> (Function, Keys) { match id { - i if i == self.ids[0] => Function::Rewrite, - i if i == self.ids[1] => Function::RewriteDirectly, - i if i == self.ids[2] => Function::Translate, - i if i == self.ids[3] => Function::TranslateDirectly, + i if i == self.ids[0] => (Function::Rewrite, self.hotkeys_keys[0].clone()), + i if i == self.ids[1] => (Function::RewriteDirectly, self.hotkeys_keys[1].clone()), + i if i == self.ids[2] => (Function::Translate, self.hotkeys_keys[2].clone()), + i if i == self.ids[3] => (Function::TranslateDirectly, self.hotkeys_keys[3].clone()), _ => unreachable!(), } } diff --git a/src/service/keyboard.rs b/src/service/keyboard.rs index c507664..8eb97c9 100644 --- a/src/service/keyboard.rs +++ b/src/service/keyboard.rs @@ -4,7 +4,7 @@ use std::{ thread, }; // self -use crate::component::keyboard::Keyboard as Kb; +use crate::component::keyboard::{Keyboard as Kb, Keys}; #[derive(Clone, Debug)] pub struct Keyboard(Sender); @@ -20,6 +20,8 @@ impl Keyboard { loop { match rx.recv().expect("receive must succeed") { Action::Copy => kb.copy().expect("keyboard action must succeed"), + Action::ReleaseKeys(keys) => + kb.release_keys(keys).expect("keyboard action must succeed"), Action::Text(text) => kb.text(&text).expect("keyboard action must succeed"), Action::Abort => return, } @@ -33,6 +35,10 @@ impl Keyboard { self.0.send(Action::Copy).expect("send must succeed"); } + pub fn release_keys(&self, keys: Keys) { + self.0.send(Action::ReleaseKeys(keys)).expect("send must succeed"); + } + pub fn text(&self, text: String) { self.0.send(Action::Text(text)).expect("send must succeed"); } @@ -45,6 +51,7 @@ impl Keyboard { #[derive(Debug)] enum Action { Copy, + ReleaseKeys(Keys), Text(String), Abort, } diff --git a/src/ui/panel/chat.rs b/src/ui/panel/chat.rs index ef0a0ee..3c86bda 100644 --- a/src/ui/panel/chat.rs +++ b/src/ui/panel/chat.rs @@ -88,6 +88,10 @@ impl UiT for Chat { if !self.shortcut.copy.triggered { if ui.add(self.shortcut.copy.copy_img.clone()).clicked() { self.shortcut.copy.triggered = true; + ctx.components + .clipboard + .set_text(&self.output) + .expect("clipboard must be available"); } } else { ui.add(self.shortcut.copy.copied_img.clone()); diff --git a/src/ui/panel/setting.rs b/src/ui/panel/setting.rs index 63d50b9..8c4b999 100644 --- a/src/ui/panel/setting.rs +++ b/src/ui/panel/setting.rs @@ -113,15 +113,7 @@ impl UiT for Setting { }); if changed { - ctx.services.chat.renew( - ctx.services.keyboard.clone(), - ctx.services.rt.as_ref().expect("runtime must exist"), - ctx.services.is_chatting.clone(), - ctx.components.setting.ai.clone(), - ctx.components.setting.chat.clone(), - ctx.state.chat.input.clone(), - ctx.state.chat.output.clone(), - ); + ctx.services.chat.renew(&ctx.components.setting); } });