Return a proper error type from set_virtual_terminal

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Unreleased
 
+- **[BREAKING CHANGE]:** Return `io::Error` from `control::set_virtual_terminal`.
+
 # 3.0.0
 - **[BREAKING CHANGE]:** Upgrade MSRV to 1.80 and remove the then unnecessary lazy_static dependency.
 

src/control.rs

@@ -28,31 +28,44 @@ use std::sync::LazyLock;
 /// ```
 #[allow(clippy::result_unit_err)]
 #[cfg(windows)]
-pub fn set_virtual_terminal(use_virtual: bool) -> Result<(), ()> {
+pub fn set_virtual_terminal(use_virtual: bool) -> Result<(), io::Error> {
+    use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE;
     use windows_sys::Win32::System::Console::{
         GetConsoleMode, GetStdHandle, SetConsoleMode, ENABLE_VIRTUAL_TERMINAL_PROCESSING,
         STD_OUTPUT_HANDLE,
     };
 
-    unsafe {
-        let handle = GetStdHandle(STD_OUTPUT_HANDLE);
-        let mut original_mode = 0;
-        GetConsoleMode(handle, &mut original_mode);
-
-        let enabled = original_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING
-            == ENABLE_VIRTUAL_TERMINAL_PROCESSING;
-
-        match (use_virtual, enabled) {
-            // not enabled, should be enabled
-            (true, false) => {
-                SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING | original_mode)
-            }
-            // already enabled, should be disabled
-            (false, true) => {
-                SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING ^ original_mode)
-            }
-            _ => 0,
-        };
+    let handle = unsafe { GetStdHandle(STD_OUTPUT_HANDLE) };
+    if handle == INVALID_HANDLE_VALUE {
+        return Err(io::Error::last_os_error());
+    }
+
+    let mut original_mode = 0;
+    // Return value of 0 means that `GetConsoleMode` failed:
+    // https://learn.microsoft.com/en-us/windows/console/getconsolemode#return-value
+    if unsafe { GetConsoleMode(handle, &mut original_mode) } == 0 {
+        return Err(io::Error::last_os_error());
+    }
+
+    let enabled =
+        original_mode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+
+    let ret = match (use_virtual, enabled) {
+        // not enabled, should be enabled
+        (true, false) => unsafe {
+            SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING | original_mode)
+        },
+        // already enabled, should be disabled
+        (false, true) => unsafe {
+            SetConsoleMode(handle, ENABLE_VIRTUAL_TERMINAL_PROCESSING ^ original_mode)
+        },
+        _ => 0,
+    };
+
+    // Return value of 0 means that `SetConsoleMode` failed:
+    // https://learn.microsoft.com/en-us/windows/console/setconsolemode#return-value
+    if ret == 0 {
+        return Err(io::Error::last_os_error());
     }
 
     Ok(())