diff --git a/CHANGELOG.md b/CHANGELOG.md index 577c8b2..7f4df9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ # Unreleased +- **[BREAKING CHANGE]:** Return `io::Error` from `control::set_virtual_terminal`. + # 3.0.0 - **[BREAKING CHANGE]:** Upgrade MSRV to 1.80 and remove the then unnecessary lazy_static dependency. diff --git a/src/control.rs b/src/control.rs index 690f270..c4f1b39 100644 --- a/src/control.rs +++ b/src/control.rs @@ -28,31 +28,44 @@ use std::sync::LazyLock; /// ``` #[allow(clippy::result_unit_err)] #[cfg(windows)] -pub fn set_virtual_terminal(use_virtual: bool) -> Result<(), ()> { +pub fn set_virtual_terminal(use_virtual: bool) -> Result<(), io::Error> { + use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; use windows_sys::Win32::System::Console::{ GetConsoleMode, GetStdHandle, SetConsoleMode, ENABLE_VIRTUAL_TERMINAL_PROCESSING, STD_OUTPUT_HANDLE, }; - unsafe { - let handle = GetStdHandle(STD_OUTPUT_HANDLE); - let mut original_mode = 0; - GetConsoleMode(handle, &mut original_mode); - - let enabled = original_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING - == ENABLE_VIRTUAL_TERMINAL_PROCESSING; - - match (use_virtual, enabled) { - // not enabled, should be enabled - (true, false) => { - SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING | original_mode) - } - // already enabled, should be disabled - (false, true) => { - SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING ^ original_mode) - } - _ => 0, - }; + let handle = unsafe { GetStdHandle(STD_OUTPUT_HANDLE) }; + if handle == INVALID_HANDLE_VALUE { + return Err(io::Error::last_os_error()); + } + + let mut original_mode = 0; + // Return value of 0 means that `GetConsoleMode` failed: + // https://learn.microsoft.com/en-us/windows/console/getconsolemode#return-value + if unsafe { GetConsoleMode(handle, &mut original_mode) } == 0 { + return Err(io::Error::last_os_error()); + } + + let enabled = + original_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == ENABLE_VIRTUAL_TERMINAL_PROCESSING; + + let ret = match (use_virtual, enabled) { + // not enabled, should be enabled + (true, false) => unsafe { + SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING | original_mode) + }, + // already enabled, should be disabled + (false, true) => unsafe { + SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING ^ original_mode) + }, + _ => 0, + }; + + // Return value of 0 means that `SetConsoleMode` failed: + // https://learn.microsoft.com/en-us/windows/console/setconsolemode#return-value + if ret == 0 { + return Err(io::Error::last_os_error()); } Ok(())