diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6eb20fd1..aa4195a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,8 +22,8 @@ jobs: uses: actions/checkout@v3 with: submodules: true - - name: Install Rust (1.83 w/ clippy) - uses: dtolnay/rust-toolchain@1.83 + - name: Install Rust (1.87 w/ clippy) + uses: dtolnay/rust-toolchain@1.87 with: components: clippy - name: Install Rust (nightly w/ rustfmt) diff --git a/docs/iamb.5 b/docs/iamb.5 index 8357f965..e7f53112 100644 --- a/docs/iamb.5 +++ b/docs/iamb.5 @@ -133,11 +133,25 @@ The room to show by default instead of the .Sy :welcome window. +.IT Sy cache_policy +Configure how downloaded files and image previews are cached internally. +Specifying this object will replace the upstream defaults and possibly lead to unconstrained cache growth. +Possible keys and their effect are documented at: +.br +.Lk https://docs.rs/matrix-sdk/latest/matrix_sdk/media/struct.MediaRetentionPolicy.html#fields + .It Sy image_preview Enable image previews and configure it. An empty object will enable the feature with default settings, omitting it will disable the feature. The available fields in this object are: .Bl -tag -width Ds +.It Sy lazy_load +If +.Sy true +(the default), download and render image previews when viewing a message with an image. +If +.Sy false , +load previews as soon as a message with an image is received. .It Sy size An optional object with .Sy width @@ -538,8 +552,6 @@ Configured as an object under the key .It Sy cache Specifies where to store assets and temporary data in. (For example, -.Sy image_preview -and .Sy logs will also go in here by default.) Defaults to @@ -555,11 +567,6 @@ Specifies where to store downloaded files. Defaults to .Ev $XDG_DOWNLOAD_DIR . -.It Sy image_previews -Specifies where to store automatically downloaded image previews. -Defaults to -.Ev ${cache}/image_preview_downloads . - .It Sy logs Specifies where to store log files. Defaults to diff --git a/src/base.rs b/src/base.rs index d191b809..4b45fb27 100644 --- a/src/base.rs +++ b/src/base.rs @@ -13,7 +13,9 @@ use std::time::{Duration, Instant}; use emojis::Emoji; use matrix_sdk::ruma::events::receipt::ReceiptThread; +use matrix_sdk::ruma::events::room::MediaSource; use matrix_sdk::ruma::room_version_rules::RedactionRules; +use matrix_sdk::ruma::OwnedMxcUri; use ratatui::{ buffer::Buffer, layout::{Alignment, Rect}, @@ -90,10 +92,9 @@ use modalkit::{ prelude::{CommandType, WordStyle}, }; -use crate::config::ImagePreviewProtocolValues; -use crate::message::ImageStatus; +use crate::config::{ImagePreviewProtocolValues, ImagePreviewSize}; use crate::notifications::NotificationHandle; -use crate::preview::{source_from_event, spawn_insert_preview}; +use crate::preview::{source_from_event, PreviewKind, PreviewManager}; use crate::{ message::{Message, MessageEvent, MessageKey, MessageTimeStamp, Messages}, worker::Requester, @@ -688,7 +689,7 @@ pub type IambResult = UIResult; /// /// The event identifier used as a key here is the ID for the reaction, and not for the message /// it's reacting to. -pub type MessageReactions = HashMap; +pub type MessageReactions = HashMap)>; /// Errors encountered during application use. #[derive(thiserror::Error, Debug)] @@ -994,22 +995,25 @@ impl RoomInfo { } /// Get the reactions and their counts for a message. - pub fn get_reactions(&self, event_id: &EventId) -> Vec<(&str, usize)> { + pub fn get_reactions(&self, event_id: &EventId) -> Vec<(&str, usize, &Option)> { if let Some(reacts) = self.reactions.get(event_id) { let mut counts = HashMap::new(); let mut seen_user_reactions = BTreeSet::new(); - for (key, user) in reacts.values() { + for (key, user, source) in reacts.values() { if !seen_user_reactions.contains(&(key, user)) { seen_user_reactions.insert((key, user)); - let count = counts.entry(key.as_str()).or_default(); - *count += 1; + let count = counts.entry(key.as_str()).or_insert((0, source)); + count.0 += 1; } } - let mut reactions = counts.into_iter().collect::>(); - reactions.sort(); + let mut reactions = counts + .into_iter() + .map(|(key, (count, source))| (key, count, source)) + .collect::>(); + reactions.sort_by_key(|item| (item.0, item.1)); reactions } else { @@ -1070,24 +1074,46 @@ impl RoomInfo { } /// Insert a reaction to a message. - pub fn insert_reaction(&mut self, react: ReactionEvent) { - match react { - MessageLikeEvent::Original(react) => { - let rel_id = react.content.relates_to.event_id; - let key = react.content.relates_to.key; + fn insert_reaction(&mut self, react: ReactionEvent, source: Option) { + let MessageLikeEvent::Original(react) = react else { + return; + }; + let rel_id = react.content.relates_to.event_id; + let key = react.content.relates_to.key; - let message = self.reactions.entry(rel_id.clone()).or_default(); - let event_id = react.event_id; - let user_id = react.sender; + let message = self.reactions.entry(rel_id.clone()).or_default(); + let event_id = react.event_id; + let user_id = react.sender; - message.insert(event_id.clone(), (key, user_id)); + message.insert(event_id.clone(), (key, user_id, source)); - let loc = EventLocation::Reaction(rel_id); - self.keys.insert(event_id, loc); - }, - MessageLikeEvent::Redacted(_) => { - return; - }, + let loc = EventLocation::Reaction(rel_id); + self.keys.insert(event_id, loc); + } + + /// Insert a reaction to a message. + pub fn insert_reaction_with_preview( + &mut self, + react: ReactionEvent, + settings: &ApplicationSettings, + previews: &mut PreviewManager, + worker: &Requester, + ) { + let MessageLikeEvent::Original(ref orig_react) = react else { + return; + }; + let image_uri = OwnedMxcUri::from(orig_react.content.relates_to.key.as_str()); + let source = if image_uri.is_valid() && settings.tunables.image_preview.is_some() { + Some(MediaSource::Plain(image_uri)) + } else { + None + }; + + self.insert_reaction(react, source.clone()); + + if let (Some(source), Some(_)) = (source, &settings.tunables.image_preview) { + let size = ImagePreviewSize { width: 2, height: 1 }; + previews.register_preview(settings, source, PreviewKind::Reaction, size, worker); } } @@ -1233,32 +1259,28 @@ impl RoomInfo { } } - /// Insert a new message event, and spawn a task for image-preview if it has an image - /// attachment. + /// Insert a new message event, and prepare for image-preview if it has an image attachment. pub fn insert_with_preview( &mut self, - room_id: OwnedRoomId, - store: AsyncProgramStore, - picker: Option, ev: RoomMessageEvent, - settings: &mut ApplicationSettings, - media: matrix_sdk::Media, + settings: &ApplicationSettings, + previews: &mut PreviewManager, + worker: &Requester, ) { - let source = picker.and_then(|_| source_from_event(&ev)); + let source = source_from_event(&ev); self.insert(ev); if let Some((event_id, source)) = source { if let (Some(msg), Some(image_preview)) = (self.get_event_mut(&event_id), &settings.tunables.image_preview) { - msg.image_preview = ImageStatus::Downloading(image_preview.size.clone()); - spawn_insert_preview( - store, - room_id, - event_id, + msg.image_preview = Some(source.clone()); + previews.register_preview( + settings, source, - media, - settings.dirs.image_previews.clone(), + PreviewKind::Message, + image_preview.size, + worker, ) } } @@ -1416,7 +1438,7 @@ impl RoomInfo { if let Some(reactions) = self.reactions.get(event_id) { reactions .values() - .any(|(annotation, user)| annotation == emoji && user == user_id) + .any(|(annotation, user, _)| annotation == emoji && user == user_id) } else { false } @@ -1601,8 +1623,8 @@ pub struct ChatStore { /// Information gathered by the background thread. pub sync_info: SyncInfo, - /// Image preview "protocol" picker. - pub picker: Option, + /// Rendered image previews. + pub previews: PreviewManager, /// Last draw time, used to match with RoomInfo's draw_last. pub draw_curr: Option, @@ -1628,7 +1650,7 @@ impl ChatStore { ChatStore { worker, settings, - picker, + previews: PreviewManager::new(picker), cmds: crate::commands::setup_commands(), emojis: emoji_map(), @@ -2196,16 +2218,17 @@ pub mod tests { for i in 0..3 { let event_id = format!("$house_{i}"); - info.insert_reaction(MessageLikeEvent::Original( - matrix_sdk::ruma::events::OriginalMessageLikeEvent { + info.insert_reaction( + MessageLikeEvent::Original(matrix_sdk::ruma::events::OriginalMessageLikeEvent { content: content.clone(), event_id: OwnedEventId::from_str(&event_id).unwrap(), sender: owned_user_id!("@foo:example.org"), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), room_id: owned_room_id!("!foo:example.org"), unsigned: MessageLikeUnsigned::new(), - }, - )); + }), + None, + ); } let content = ReactionEventContent::new(Annotation::new( @@ -2215,36 +2238,40 @@ pub mod tests { for i in 0..2 { let event_id = format!("$smile_{i}"); - info.insert_reaction(MessageLikeEvent::Original( - matrix_sdk::ruma::events::OriginalMessageLikeEvent { + info.insert_reaction( + MessageLikeEvent::Original(matrix_sdk::ruma::events::OriginalMessageLikeEvent { content: content.clone(), event_id: OwnedEventId::from_str(&event_id).unwrap(), sender: owned_user_id!("@foo:example.org"), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), room_id: owned_room_id!("!foo:example.org"), unsigned: MessageLikeUnsigned::new(), - }, - )); + }), + None, + ); } for i in 2..4 { let event_id = format!("$smile_{i}"); - info.insert_reaction(MessageLikeEvent::Original( - matrix_sdk::ruma::events::OriginalMessageLikeEvent { + info.insert_reaction( + MessageLikeEvent::Original(matrix_sdk::ruma::events::OriginalMessageLikeEvent { content: content.clone(), event_id: OwnedEventId::from_str(&event_id).unwrap(), sender: owned_user_id!("@bar:example.org"), origin_server_ts: MilliSecondsSinceUnixEpoch::now(), room_id: owned_room_id!("!foo:example.org"), unsigned: MessageLikeUnsigned::new(), - }, - )); + }), + None, + ); } - assert_eq!(info.get_reactions(&owned_event_id!("$my_reaction")), vec![ - ("🏠", 1), - ("🙂", 2) - ]); + let reacts: Vec<_> = info + .get_reactions(&owned_event_id!("$my_reaction")) + .into_iter() + .map(|(key, count, _)| (key, count)) + .collect(); + assert_eq!(reacts, vec![("🏠", 1), ("🙂", 2)]); } #[test] diff --git a/src/config.rs b/src/config.rs index e7a4a47b..01275a09 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,6 +12,7 @@ use std::process; use clap::Parser; use matrix_sdk::authentication::matrix::MatrixSession; +use matrix_sdk::media::MediaRetentionPolicy; use matrix_sdk::ruma::{OwnedDeviceId, OwnedRoomAliasId, OwnedRoomId, OwnedUserId, UserId}; use ratatui::style::{Color, Modifier as StyleModifier, Style}; use ratatui::text::Span; @@ -489,12 +490,14 @@ pub struct Notifications { #[derive(Clone)] pub struct ImagePreviewValues { + pub lazy_load: bool, pub size: ImagePreviewSize, pub protocol: Option, } #[derive(Clone, Default, Deserialize)] pub struct ImagePreview { + pub lazy_load: Option, pub size: Option, pub protocol: Option, } @@ -502,13 +505,14 @@ pub struct ImagePreview { impl ImagePreview { fn values(self) -> ImagePreviewValues { ImagePreviewValues { + lazy_load: self.lazy_load.unwrap_or(true), size: self.size.unwrap_or_default(), protocol: self.protocol, } } } -#[derive(Clone, Deserialize)] +#[derive(Clone, Copy, Deserialize, Debug)] pub struct ImagePreviewSize { pub width: usize, pub height: usize, @@ -581,6 +585,7 @@ pub struct TunableValues { pub user_gutter_width: usize, pub external_edit_file_suffix: String, pub tabstop: usize, + pub cache_policy: MediaRetentionPolicy, } #[derive(Clone, Default, Deserialize)] @@ -609,6 +614,7 @@ pub struct Tunables { pub user_gutter_width: Option, pub external_edit_file_suffix: Option, pub tabstop: Option, + pub cache_policy: Option, } impl Tunables { @@ -643,6 +649,7 @@ impl Tunables { .external_edit_file_suffix .or(other.external_edit_file_suffix), tabstop: self.tabstop.or(other.tabstop), + cache_policy: self.cache_policy.or(other.cache_policy), } } @@ -673,6 +680,7 @@ impl Tunables { .external_edit_file_suffix .unwrap_or_else(|| ".md".to_string()), tabstop: self.tabstop.unwrap_or(4), + cache_policy: self.cache_policy.unwrap_or_default(), } } } @@ -683,19 +691,17 @@ pub struct DirectoryValues { pub data: PathBuf, pub logs: PathBuf, pub downloads: Option, - pub image_previews: PathBuf, } impl DirectoryValues { fn create_dir_all(&self) -> std::io::Result<()> { use std::fs::create_dir_all; - let Self { cache, data, logs, downloads, image_previews } = self; + let Self { cache, data, logs, downloads } = self; create_dir_all(cache)?; create_dir_all(data)?; create_dir_all(logs)?; - create_dir_all(image_previews)?; if let Some(downloads) = downloads { create_dir_all(downloads)?; @@ -711,7 +717,6 @@ pub struct Directories { pub data: Option, pub logs: Option, pub downloads: Option, - pub image_previews: Option, } impl Directories { @@ -721,7 +726,6 @@ impl Directories { data: self.data.or(other.data), logs: self.logs.or(other.logs), downloads: self.downloads.or(other.downloads), - image_previews: self.image_previews.or(other.image_previews), } } @@ -776,20 +780,7 @@ impl Directories { }) .or_else(dirs::download_dir); - let image_previews = self - .image_previews - .map(|dir| { - let dir = shellexpand::full(&dir) - .expect("unable to expand shell variables in dirs.cache"); - Path::new(dir.as_ref()).to_owned() - }) - .unwrap_or_else(|| { - let mut dir = cache.clone(); - dir.push("image_preview_downloads"); - dir - }); - - DirectoryValues { cache, data, logs, downloads, image_previews } + DirectoryValues { cache, data, logs, downloads } } } diff --git a/src/message/mod.rs b/src/message/mod.rs index 718c7a68..00b238f2 100644 --- a/src/message/mod.rs +++ b/src/message/mod.rs @@ -11,6 +11,7 @@ use std::ops::{Deref, DerefMut}; use chrono::{DateTime, Local as LocalTz}; use humansize::{format_size, DECIMAL}; use matrix_sdk::ruma::events::receipt::ReceiptThread; +use matrix_sdk::ruma::events::room::MediaSource; use matrix_sdk::ruma::room_version_rules::RedactionRules; use serde_json::json; use unicode_width::UnicodeWidthStr; @@ -58,6 +59,7 @@ use modalkit::prelude::*; use ratatui_image::protocol::Protocol; use crate::config::ImagePreviewSize; +use crate::preview::{ImageStatus, PreviewKind, PreviewManager}; use crate::{ base::RoomInfo, config::ApplicationSettings, @@ -731,6 +733,7 @@ impl<'a> MessageFormatter<'a> { text: &mut Text<'a>, info: &'a RoomInfo, settings: &'a ApplicationSettings, + previews: &'a PreviewManager, ) -> Option> { let reply_style = if settings.tunables.message_user_color { style.patch(settings.get_user_color(&msg.sender)) @@ -740,7 +743,7 @@ impl<'a> MessageFormatter<'a> { let width = self.width(); let w = width.saturating_sub(2); - let (mut replied, proto) = msg.show_msg(w, reply_style, true, settings); + let (mut replied, proto) = msg.show_msg(w, reply_style, true, settings, previews); let mut sender = msg.sender_span(info, self.settings); let sender_width = UnicodeWidthStr::width(sender.content.as_ref()); let trailing = w.saturating_sub(sender_width + 1); @@ -777,16 +780,37 @@ impl<'a> MessageFormatter<'a> { proto } - fn push_reactions(&mut self, counts: Vec<(&'a str, usize)>, style: Style, text: &mut Text<'a>) { + fn push_reactions( + &mut self, + counts: Vec<(&'a str, usize, &'a Option)>, + style: Style, + text: &mut Text<'a>, + settings: &ApplicationSettings, + previews: &'a PreviewManager, + ) -> Vec> { let mut emojis = printer::TextPrinter::new(self.width(), style, false, self.settings); let mut reactions = 0; + let mut protos = Vec::new(); - for (key, count) in counts { + for (key, count, source) in counts { if reactions != 0 { emojis.push_str(" ", style); } - let name = if self.settings.tunables.reaction_shortcode_display { + let proto = match source + .as_ref() + .and_then(|source| previews.get(source, PreviewKind::Reaction)) + { + Some(ImageStatus::Loaded(backend)) => Some(Some(backend)), + // Use empty space as placeholder + Some(ImageStatus::Queued(_)) | Some(ImageStatus::Downloading(_)) => Some(None), + // Fall back to text + None | Some(ImageStatus::Error(_)) => None, + }; + + let name = if proto.is_some() { + " " + } else if self.settings.tunables.reaction_shortcode_display { if let Some(emoji) = emojis::get(key) { if let Some(short) = emoji.shortcode() { short @@ -805,6 +829,13 @@ impl<'a> MessageFormatter<'a> { }; emojis.push_str("[", style); + if let Some(Some(proto)) = proto { + let (x, y) = emojis.curosor_pos(); + let y = (y + text.lines.len()) as u16; + let x = x as u16 + self.cols.user_gutter_width(settings); + + protos.push((proto, x, y)); + } emojis.push_str(name, style); emojis.push_str(" ", style); emojis.push_span_nobreak(Span::styled(count.to_string(), style)); @@ -816,6 +847,8 @@ impl<'a> MessageFormatter<'a> { if reactions > 0 { self.push_text(emojis.finish(), style, text); } + + protos } fn push_thread_reply_count(&mut self, len: usize, text: &mut Text<'a>) { @@ -841,20 +874,13 @@ impl<'a> MessageFormatter<'a> { } } -pub enum ImageStatus { - None, - Downloading(ImagePreviewSize), - Loaded(Protocol), - Error(String), -} - pub struct Message { pub event: MessageEvent, pub sender: OwnedUserId, pub timestamp: MessageTimeStamp, pub downloaded: bool, pub html: Option, - pub image_preview: ImageStatus, + pub image_preview: Option, } impl Message { @@ -868,7 +894,7 @@ impl Message { timestamp, downloaded, html, - image_preview: ImageStatus::None, + image_preview: None, } } @@ -893,7 +919,7 @@ impl Message { } } - fn thread_root(&self) -> Option { + pub fn thread_root(&self) -> Option { let content = match &self.event { MessageEvent::EncryptedOriginal(_) => return None, MessageEvent::EncryptedRedacted(_) => return None, @@ -1000,7 +1026,8 @@ impl Message { vwctx: &ViewportContext, info: &'a RoomInfo, settings: &'a ApplicationSettings, - ) -> (Text<'a>, [Option>; 2]) { + previews: &'a PreviewManager, + ) -> (Text<'a>, Vec>) { let width = vwctx.get_width(); let style = self.get_render_style(selected, settings); @@ -1008,28 +1035,35 @@ impl Message { let mut text = Text::default(); let width = fmt.width(); + let mut protos = Vec::new(); + // Show the message that this one replied to, if any. let reply = self .reply_to() .or_else(|| self.thread_root()) .and_then(|e| info.get_event(&e)); - let proto_reply = reply.as_ref().and_then(|r| { + if let Some(reply) = reply { // Format the reply header, push it into the `Text` buffer, and get any image. - fmt.push_in_reply(r, style, &mut text, info, settings) - }); + if let Some(proto) = + fmt.push_in_reply(reply, style, &mut text, info, settings, previews) + { + protos.push(proto); + } + } // Now show the message contents, and the inlined reply if we couldn't find it above. - let (msg, proto) = self.show_msg(width, style, reply.is_some(), settings); + let (msg, proto) = self.show_msg(width, style, reply.is_some(), settings, previews); // Given our text so far, determine the image offset. - let proto_main = proto.map(|p| { + if let Some(p) = proto { let y_off = text.lines.len() as u16; let x_off = fmt.cols.user_gutter_width(settings); // Adjust y_off by 1 if a date was printed before the message to account for // the extra line we're going to print. let y_off = if fmt.date.is_some() { y_off + 1 } else { y_off }; - (p, x_off, y_off) - }); + + protos.push((p, x_off, y_off)); + } fmt.push_text(msg, style, &mut text); @@ -1040,14 +1074,15 @@ impl Message { if settings.tunables.reaction_display { let reactions = info.get_reactions(self.event.event_id()); - fmt.push_reactions(reactions, style, &mut text); + let react_protos = fmt.push_reactions(reactions, style, &mut text, settings, previews); + protos.extend(react_protos); } if let Some(thread) = info.get_thread(Some(self.event.event_id())) { fmt.push_thread_reply_count(thread.len(), &mut text); } - (text, [proto_main, proto_reply]) + (text, protos) } pub fn show<'a>( @@ -1057,8 +1092,9 @@ impl Message { vwctx: &ViewportContext, info: &'a RoomInfo, settings: &'a ApplicationSettings, + previews: &'a PreviewManager, ) -> Text<'a> { - self.show_with_preview(prev, selected, vwctx, info, settings).0 + self.show_with_preview(prev, selected, vwctx, info, settings, previews).0 } fn show_msg<'a>( @@ -1067,6 +1103,7 @@ impl Message { style: Style, hide_reply: bool, settings: &'a ApplicationSettings, + previews: &'a PreviewManager, ) -> (Text<'a>, Option<&'a Protocol>) { if let Some(html) = &self.html { (html.to_text(width, style, hide_reply, settings), None) @@ -1081,16 +1118,23 @@ impl Message { } let mut proto = None; - let placeholder = match &self.image_preview { - ImageStatus::None => None, - ImageStatus::Downloading(image_preview_size) => { + let placeholder = match self + .image_preview + .as_ref() + .and_then(|source| previews.get(source, PreviewKind::Message)) + { + None => None, + Some(ImageStatus::Queued(image_preview_size)) => { + placeholder_frame(Some("Queued..."), width, image_preview_size) + }, + Some(ImageStatus::Downloading(image_preview_size)) => { placeholder_frame(Some("Downloading..."), width, image_preview_size) }, - ImageStatus::Loaded(backend) => { + Some(ImageStatus::Loaded(backend)) => { proto = Some(backend); placeholder_frame(Some("No Space..."), width, &backend.area().into()) }, - ImageStatus::Error(err) => Some(format!("[Image error: {err}]\n")), + Some(ImageStatus::Error(err)) => Some(format!("[Image error: {err}]\n")), }; if let Some(placeholder) = placeholder { @@ -1143,7 +1187,7 @@ impl Message { self.event.redact(redaction, rules); self.html = None; self.downloaded = false; - self.image_preview = ImageStatus::None; + self.image_preview = None; } } diff --git a/src/message/printer.rs b/src/message/printer.rs index 34187521..b2034d44 100644 --- a/src/message/printer.rs +++ b/src/message/printer.rs @@ -57,6 +57,11 @@ impl<'a> TextPrinter<'a> { } } + /// The position where the next text will be printed. (x, y) + pub fn curosor_pos(&self) -> (usize, usize) { + (self.curr_width, self.text.lines.len().saturating_sub(1)) + } + /// Configure the alignment for each line. pub fn align(mut self, alignment: Alignment) -> Self { self.alignment = alignment; diff --git a/src/preview.rs b/src/preview.rs index f2c620a5..725843f9 100644 --- a/src/preview.rs +++ b/src/preview.rs @@ -1,11 +1,7 @@ -use std::{ - fs::File, - io::{Read, Write}, - path::{Path, PathBuf}, -}; +use std::{collections::HashMap, sync::Arc}; use matrix_sdk::{ - media::{MediaFormat, MediaRequestParameters}, + media::{MediaFormat, MediaRequestParameters, UniqueKey}, ruma::{ events::{ room::{ @@ -15,19 +11,110 @@ use matrix_sdk::{ MessageLikeEvent, }, OwnedEventId, - OwnedRoomId, }, Media, }; use ratatui::layout::Rect; -use ratatui_image::Resize; +use ratatui_image::{picker::Picker, protocol::Protocol, Resize}; +use tokio::sync::Semaphore; use crate::{ - base::{AsyncProgramStore, ChatStore, IambError}, - config::ImagePreviewSize, - message::ImageStatus, + base::{AsyncProgramStore, IambError}, + config::{ApplicationSettings, ImagePreviewSize}, + worker::Requester, }; +pub enum ImageStatus { + Queued(ImagePreviewSize), + Downloading(ImagePreviewSize), + Loaded(Protocol), + Error(String), +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum PreviewKind { + Message, + Reaction, +} + +pub struct PreviewManager { + /// Image preview "protocol" picker. + picker: Option>, + + /// Permits for rendering images in background thread. + permits: Arc, + + /// Indexed by [`MediaSource::unique_key`] + previews: HashMap<(String, PreviewKind), ImageStatus>, +} + +impl PreviewManager { + pub fn new(picker: Option) -> Self { + Self { + picker: picker.map(Into::into), + permits: Arc::new(Semaphore::new(2)), + previews: Default::default(), + } + } + + pub fn get(&self, source: &MediaSource, kind: PreviewKind) -> Option<&ImageStatus> { + self.previews.get(&(source.unique_key(), kind)) + } + + fn insert(&mut self, key: String, kind: PreviewKind, status: ImageStatus) { + self.previews.insert((key, kind), status); + } + + /// Queue download and preparation of preview + pub fn load(&mut self, source: &MediaSource, kind: PreviewKind, worker: &Requester) { + let Some(status) = self.previews.get_mut(&(source.unique_key(), kind)) else { + return; + }; + let Some(picker) = &self.picker else { return }; + + if let ImageStatus::Queued(size) = status { + let size = *size; + *status = ImageStatus::Downloading(size); + + worker.load_image( + source.to_owned(), + kind, + size.to_owned(), + Arc::clone(picker), + Arc::clone(&self.permits), + ); + } + } + + pub fn register_preview( + &mut self, + settings: &ApplicationSettings, + source: MediaSource, + kind: PreviewKind, + size: ImagePreviewSize, + worker: &Requester, + ) { + if self.picker.is_none() { + return; + } + + let key = (source.unique_key(), kind); + if self.previews.contains_key(&key) { + return; + } + self.previews.insert(key, ImageStatus::Queued(size)); + + if settings + .tunables + .image_preview + .as_ref() + .is_some_and(|setting| !setting.lazy_load) + { + self.load(&source, kind, worker); + } + } +} + pub fn source_from_event( ev: &MessageLikeEvent, ) -> Option<(OwnedEventId, MediaSource)> { @@ -50,126 +137,55 @@ impl From for ImagePreviewSize { } } -/// Download and prepare the preview, and then lock the store to insert it. -pub fn spawn_insert_preview( +pub async fn load_image( store: AsyncProgramStore, - room_id: OwnedRoomId, - event_id: OwnedEventId, - source: MediaSource, media: Media, - cache_dir: PathBuf, + source: MediaSource, + kind: PreviewKind, + picker: Arc, + permits: Arc, + size: ImagePreviewSize, ) { - tokio::spawn(async move { - let img = download_or_load(event_id.to_owned(), source, media, cache_dir) + async fn load_image_inner( + media: Media, + source: MediaSource, + picker: Arc, + permits: Arc, + size: ImagePreviewSize, + ) -> Result { + let reader = media + .get_media_content(&MediaRequestParameters { source, format: MediaFormat::File }, true) .await .map(std::io::Cursor::new) .map(image::ImageReader::new) .map_err(IambError::Matrix) - .and_then(|reader| reader.with_guessed_format().map_err(IambError::IOError)) - .and_then(|reader| reader.decode().map_err(IambError::Image)); - - match img { - Err(err) => { - try_set_msg_preview_error( - &mut store.lock().await.application, - room_id, - event_id, - err, - ); - }, - Ok(img) => { - let mut locked = store.lock().await; - let ChatStore { rooms, picker, settings, .. } = &mut locked.application; - - match picker - .as_mut() - .ok_or_else(|| IambError::Preview("Picker is empty".to_string())) - .and_then(|picker| { - Ok(( - picker, - rooms - .get_or_default(room_id.clone()) - .get_event_mut(&event_id) - .ok_or_else(|| { - IambError::Preview("Message not found".to_string()) - })?, - settings.tunables.image_preview.clone().ok_or_else(|| { - IambError::Preview("image_preview settings not found".to_string()) - })?, - )) - }) - .and_then(|(picker, msg, image_preview)| { - picker - .new_protocol(img, image_preview.size.into(), Resize::Fit(None)) - .map_err(|err| IambError::Preview(format!("{err:?}"))) - .map(|backend| (backend, msg)) - }) { - Err(err) => { - try_set_msg_preview_error(&mut locked.application, room_id, event_id, err); - }, - Ok((backend, msg)) => { - msg.image_preview = ImageStatus::Loaded(backend); - }, - } - }, - } - }); -} + .and_then(|reader| reader.with_guessed_format().map_err(IambError::IOError))?; -fn try_set_msg_preview_error( - application: &mut ChatStore, - room_id: OwnedRoomId, - event_id: OwnedEventId, - err: IambError, -) { - let rooms = &mut application.rooms; - - match rooms - .get_or_default(room_id.clone()) - .get_event_mut(&event_id) - .ok_or_else(|| IambError::Preview("Message not found".to_string())) - { - Ok(msg) => msg.image_preview = ImageStatus::Error(format!("{err:?}")), - Err(err) => { - tracing::error!( - "Failed to set error on msg.image_backend for event {}, room {}: {}", - event_id, - room_id, - err - ) - }, - } -} + let image = reader.decode().map_err(IambError::Image)?; -async fn download_or_load( - event_id: OwnedEventId, - source: MediaSource, - media: Media, - mut cache_path: PathBuf, -) -> Result, matrix_sdk::Error> { - cache_path.push(Path::new(event_id.localpart())); - - match File::open(&cache_path) { - Ok(mut f) => { - let mut buffer = Vec::new(); - f.read_to_end(&mut buffer)?; - Ok(buffer) - }, - Err(_) => { - media - .get_media_content( - &MediaRequestParameters { source, format: MediaFormat::File }, - true, - ) - .await - .and_then(|buffer| { - if let Err(err) = - File::create(&cache_path).and_then(|mut f| f.write_all(&buffer)) - { - return Err(err.into()); - } - Ok(buffer) - }) - }, + let permit = permits + .acquire() + .await + .map_err(|err| IambError::Preview(err.to_string()))?; + + let handle = tokio::task::spawn_blocking(move || { + picker + .new_protocol(image, size.into(), Resize::Fit(None)) + .map_err(|err| IambError::Preview(err.to_string())) + }); + + let image = handle.await.map_err(|err| IambError::Preview(err.to_string()))??; + std::mem::drop(permit); + + Ok(ImageStatus::Loaded(image)) } + let key = source.unique_key(); + + let status = match load_image_inner(media, source, picker, permits, size).await { + Ok(status) => status, + Err(err) => ImageStatus::Error(format!("{err:?}")), + }; + + let mut locked = store.lock().await; + locked.application.previews.insert(key, kind, status); } diff --git a/src/tests.rs b/src/tests.rs index e9b05021..4a525374 100644 --- a/src/tests.rs +++ b/src/tests.rs @@ -163,7 +163,6 @@ pub fn mock_dirs() -> DirectoryValues { data: PathBuf::new(), logs: PathBuf::new(), downloads: None, - image_previews: PathBuf::new(), } } @@ -202,6 +201,7 @@ pub fn mock_tunables() -> TunableValues { image_preview: None, user_gutter_width: 30, tabstop: 4, + cache_policy: Default::default(), } } diff --git a/src/windows/room/chat.rs b/src/windows/room/chat.rs index 3a518aa0..9306c29d 100644 --- a/src/windows/room/chat.rs +++ b/src/windows/room/chat.rs @@ -506,21 +506,22 @@ impl ChatState { None => return Ok(None), }; - let reactions = reactions.iter().filter_map(|(event_id, (reaction, user_id))| { - if user_id != &settings.profile.user_id { - return None; - } + let reactions = + reactions.iter().filter_map(|(event_id, (reaction, user_id, _))| { + if user_id != &settings.profile.user_id { + return None; + } - if let Some(emoji) = &emoji { - if emoji == reaction { - return Some(event_id); + if let Some(emoji) = &emoji { + if emoji == reaction { + return Some(event_id); + } else { + return None; + } } else { - return None; + return Some(event_id); } - } else { - return Some(event_id); - } - }); + }); for reaction in reactions { let _ = room.redact(reaction, None, None).await.map_err(IambError::from)?; diff --git a/src/windows/room/scrollback.rs b/src/windows/room/scrollback.rs index 02340573..c6758ab7 100644 --- a/src/windows/room/scrollback.rs +++ b/src/windows/room/scrollback.rs @@ -55,6 +55,7 @@ use crate::{ }, config::ApplicationSettings, message::{Message, MessageCursor, MessageKey, Messages}, + preview::{PreviewKind, PreviewManager}, }; fn no_msgs() -> EditError { @@ -270,6 +271,7 @@ impl ScrollbackState { pos: MovePosition, info: &RoomInfo, settings: &ApplicationSettings, + previews: &PreviewManager, ) { let Some(thread) = self.get_thread(info) else { return; @@ -292,7 +294,8 @@ impl ScrollbackState { for (key, item) in thread.range(..=&idx).rev() { let sel = selidx == key; let prev = prevmsg(key, thread); - let len = item.show(prev, sel, &self.viewctx, info, settings).lines.len(); + let len = + item.show(prev, sel, &self.viewctx, info, settings, previews).lines.len(); if key == &idx { lines += len / 2; @@ -315,7 +318,8 @@ impl ScrollbackState { for (key, item) in thread.range(..=&idx).rev() { let sel = key == selidx; let prev = prevmsg(key, thread); - let len = item.show(prev, sel, &self.viewctx, info, settings).lines.len(); + let len = + item.show(prev, sel, &self.viewctx, info, settings, previews).lines.len(); lines += len; @@ -338,7 +342,12 @@ impl ScrollbackState { self.jumped.push(self.cursor.clone()); } - fn shift_cursor(&mut self, info: &RoomInfo, settings: &ApplicationSettings) { + fn shift_cursor( + &mut self, + info: &RoomInfo, + settings: &ApplicationSettings, + previews: &PreviewManager, + ) { let Some(thread) = self.get_thread(info) else { return; }; @@ -368,7 +377,10 @@ impl ScrollbackState { break; } - lines += item.show(prev, false, &self.viewctx, info, settings).height().max(1); + lines += item + .show(prev, false, &self.viewctx, info, settings, previews) + .height() + .max(1); if lines >= self.viewctx.get_height() { // We've reached the end of the viewport; move cursor into it. @@ -1067,6 +1079,7 @@ impl ScrollActions for ScrollbackState { ) -> EditResult { let info = store.application.rooms.get_or_default(self.room_id.clone()); let settings = &store.application.settings; + let previews = &store.application.previews; let mut corner = self.viewctx.corner.clone(); let thread = self.get_thread(info).ok_or_else(no_msgs)?; @@ -1094,7 +1107,7 @@ impl ScrollActions for ScrollbackState { for (key, item) in thread.range(..=&corner_key).rev() { let sel = key == cursor_key; let prev = prevmsg(key, thread); - let txt = item.show(prev, sel, &self.viewctx, info, settings); + let txt = item.show(prev, sel, &self.viewctx, info, settings, previews); let len = txt.height().max(1); let max = len.saturating_sub(1); @@ -1122,7 +1135,7 @@ impl ScrollActions for ScrollbackState { for (key, item) in thread.range(&corner_key..) { let sel = key == cursor_key; - let txt = item.show(prev, sel, &self.viewctx, info, settings); + let txt = item.show(prev, sel, &self.viewctx, info, settings, previews); let len = txt.height().max(1); let max = len.saturating_sub(1); @@ -1160,7 +1173,7 @@ impl ScrollActions for ScrollbackState { } self.viewctx.corner = corner; - self.shift_cursor(info, settings); + self.shift_cursor(info, settings, previews); Ok(None) } @@ -1182,10 +1195,11 @@ impl ScrollActions for ScrollbackState { Axis::Vertical => { let info = store.application.rooms.get_or_default(self.room_id.clone()); let settings = &store.application.settings; + let previews = &store.application.previews; let thread = self.get_thread(info).ok_or_else(no_msgs)?; if let Some(key) = self.cursor.to_key(thread).cloned() { - self.scrollview(key, pos, info, settings); + self.scrollview(key, pos, info, settings, previews); } Ok(None) @@ -1345,10 +1359,47 @@ impl StatefulWidget for Scrollback<'_> { let mut sawit = false; let mut prev = prevmsg(&corner_key, thread); + // load image previews + for ((_, event_id), item) in thread.range(&corner_key..).rev() { + if let Some(source) = &item.image_preview { + self.store.application.previews.load( + source, + PreviewKind::Message, + &self.store.application.worker, + ); + } + let reply = item + .reply_to() + .or_else(|| item.thread_root()) + .and_then(|e| info.get_event(&e)) + .and_then(|msg| msg.image_preview.as_ref()); + if let Some(source) = reply { + self.store.application.previews.load( + source, + PreviewKind::Message, + &self.store.application.worker, + ); + } + let reactions = info.reactions.get(event_id).map(|reactions| { + reactions.iter().filter_map(|(_, (_, _, source))| source.as_ref()) + }); + if let Some(reactions) = reactions { + for source in reactions { + self.store.application.previews.load( + source, + PreviewKind::Reaction, + &self.store.application.worker, + ); + } + } + } + + let previews = &self.store.application.previews; for (key, item) in thread.range(&corner_key..) { let sel = key == cursor_key; - let (txt, [mut msg_preview, mut reply_preview]) = - item.show_with_preview(prev, foc && sel, &state.viewctx, info, settings); + + let (txt, mut msg_previews) = + item.show_with_preview(prev, foc && sel, &state.viewctx, info, settings, previews); let incomplete_ok = !full || !sel; @@ -1364,17 +1415,9 @@ impl StatefulWidget for Scrollback<'_> { continue; } - // Only take the preview into the matching row number. - // `reply` and `msg` previews are on rows, - // so an `or` works to pick the one that matches (if any) - let line_preview = match msg_preview { - Some((_, _, y)) if y as usize == row => msg_preview.take(), - _ => None, - } - .or(match reply_preview { - Some((_, _, y)) if y as usize == row => reply_preview.take(), - _ => None, - }); + // Only take the previews into the matching row number. + let line_preview: Vec<_> = + msg_previews.extract_if(.., |(_, _, y)| *y as usize == row).collect(); lines.push((key, row, line, line_preview)); sawit |= sel; @@ -1399,9 +1442,9 @@ impl StatefulWidget for Scrollback<'_> { let mut image_previews = vec![]; for ((_, _), _, txt, line_preview) in lines.into_iter() { let _ = buf.set_line(x, y, &txt, area.width); - if let Some((backend, msg_x, _)) = line_preview { - image_previews.push((x + msg_x, y, backend)); - } + image_previews.extend( + line_preview.into_iter().map(|(backend, msg_x, _)| (x + msg_x, y, backend)), + ); y += 1; } diff --git a/src/worker.rs b/src/worker.rs index f3c9791a..3c631621 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -13,6 +13,8 @@ use std::time::{Duration, Instant}; use futures::{stream::FuturesUnordered, StreamExt}; use gethostname::gethostname; +use matrix_sdk::ruma::events::room::MediaSource; +use ratatui_image::picker::Picker; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tokio::sync::Semaphore; use tokio::task::JoinHandle; @@ -89,7 +91,9 @@ use modalkit::errors::UIError; use modalkit::prelude::{EditInfo, InfoMessage}; use crate::base::MessageNeed; +use crate::config::ImagePreviewSize; use crate::notifications::register_notifications; +use crate::preview::PreviewKind; use crate::{ base::{ AsyncProgramStore, @@ -256,11 +260,10 @@ async fn run_plan(client: &Client, store: &AsyncProgramStore, plan: Plan, permit Plan::Messages(room_id, fetch_id, message_need) => { let limit = MIN_MSG_LOAD; let client = client.clone(); - let store_clone = store.clone(); let res = load_older_one(&client, &room_id, fetch_id, limit).await; let mut locked = store.lock().await; - load_insert(room_id, res, locked.deref_mut(), store_clone, message_need); + load_insert(room_id, res, locked.deref_mut(), message_need); }, Plan::Members(room_id) => { let res = members_load(client, &room_id).await; @@ -322,13 +325,11 @@ fn load_insert( room_id: OwnedRoomId, res: MessageFetchResult, locked: &mut ProgramStore, - store: AsyncProgramStore, message_needs: Vec, ) { - let ChatStore { presences, rooms, worker, picker, settings, .. } = &mut locked.application; + let ChatStore { presences, rooms, previews, settings, worker, .. } = &mut locked.application; let info = rooms.get_or_default(room_id.clone()); info.fetching = false; - let client = &worker.client; match res { Ok((fetch_id, msgs)) => { @@ -345,17 +346,10 @@ fn load_insert( info.insert_encrypted(msg); }, AnyTimelineEvent::MessageLike(AnyMessageLikeEvent::RoomMessage(msg)) => { - info.insert_with_preview( - room_id.clone(), - store.clone(), - picker.clone(), - msg, - settings, - client.media(), - ); + info.insert_with_preview(msg, settings, previews, worker); }, AnyTimelineEvent::MessageLike(AnyMessageLikeEvent::Reaction(ev)) => { - info.insert_reaction(ev); + info.insert_reaction_with_preview(ev, settings, previews, worker); }, AnyTimelineEvent::MessageLike(_) => { continue; @@ -637,6 +631,7 @@ pub enum WorkerTask { TypingNotice(OwnedRoomId), Verify(VerifyAction, SasVerification, ClientReply>), VerifyRequest(OwnedUserId, ClientReply>), + LoadImage(MediaSource, PreviewKind, ImagePreviewSize, Arc, Arc), } impl Debug for WorkerTask { @@ -700,6 +695,15 @@ impl Debug for WorkerTask { .field(&format_args!("_")) .finish() }, + WorkerTask::LoadImage(source, kind, size, _, _) => { + f.debug_tuple("WorkerTask::RenderImage") + .field(source) + .field(kind) + .field(size) + .field(&format_args!("_")) + .field(&format_args!("_")) + .finish() + }, } } } @@ -720,7 +724,10 @@ async fn create_client_inner( .build() .unwrap(); - let req_config = RequestConfig::new().timeout(req_timeout).max_retry_time(req_timeout); + let req_config = RequestConfig::new() + .timeout(req_timeout) + .max_retry_time(req_timeout) + .retry_limit(8); // Set up the Matrix client for the selected profile. let builder = Client::builder() @@ -752,7 +759,15 @@ pub async fn create_client(settings: &ApplicationSettings) -> Client { res => res, }; - res.expect("Failed to instantiate client") + let client = res.expect("Failed to instantiate client"); + + client + .media() + .set_media_retention_policy(settings.tunables.cache_policy) + .await + .expect("Failed to set cache policy"); + + client } #[derive(Clone)] @@ -845,6 +860,19 @@ impl Requester { return response.recv(); } + + pub fn load_image( + &self, + source: MediaSource, + kind: PreviewKind, + size: ImagePreviewSize, + picker: Arc, + permits: Arc, + ) { + self.tx + .send(WorkerTask::LoadImage(source, kind, size, picker, permits)) + .unwrap(); + } } pub struct ClientWorker { @@ -853,6 +881,9 @@ pub struct ClientWorker { client: Client, load_handle: Option>, sync_handle: Option>, + + /// Take care when locking since worker commands are sent with the lock already hold + store: Option, } impl ClientWorker { @@ -865,6 +896,7 @@ impl ClientWorker { client: client.clone(), load_handle: None, sync_handle: None, + store: None, }; tokio::spawn(async move { @@ -938,6 +970,18 @@ impl ClientWorker { assert!(self.initialized); reply.send(self.verify_request(user_id).await); }, + WorkerTask::LoadImage(source, kind, size, picker, permits) => { + assert!(self.initialized); + tokio::spawn(crate::preview::load_image( + self.store.clone().unwrap(), + self.client.media(), + source, + kind, + picker, + permits, + size, + )); + }, } } @@ -1012,20 +1056,14 @@ impl ClientWorker { let sender = ev.sender().to_owned(); let _ = locked.application.presences.get_or_default(sender); - let ChatStore { rooms, picker, settings, .. } = &mut locked.application; + let ChatStore { rooms, previews, settings, worker, .. } = + &mut locked.application; let info = rooms.get_or_default(room_id.to_owned()); update_event_receipts(info, &room, ev.event_id()).await; let full_ev = ev.into_full_event(room_id.to_owned()); - info.insert_with_preview( - room_id.to_owned(), - store.clone(), - picker.clone(), - full_ev, - settings, - client.media(), - ); + info.insert_with_preview(full_ev, settings, previews, worker); } }, ); @@ -1042,9 +1080,18 @@ impl ClientWorker { let sender = ev.sender().to_owned(); let _ = locked.application.presences.get_or_default(sender); - let info = locked.application.get_room_info(room_id.to_owned()); + let ChatStore { rooms, previews, settings, worker, .. } = + &mut locked.application; + let info = rooms.get_or_default(room_id.to_owned()); + update_event_receipts(info, &room, ev.event_id()).await; - info.insert_reaction(ev.into_full_event(room_id.to_owned())); + + info.insert_reaction_with_preview( + ev.into_full_event(room_id.to_owned()), + settings, + previews, + worker, + ); } }, ); @@ -1255,6 +1302,8 @@ impl ClientWorker { }, ); + self.store = Some(store.clone()); + self.load_handle = tokio::spawn({ let client = self.client.clone(); let settings = self.settings.clone();