diff --git a/deltachat-jsonrpc/src/api.rs b/deltachat-jsonrpc/src/api.rs index 8eed359635..abc99e93f0 100644 --- a/deltachat-jsonrpc/src/api.rs +++ b/deltachat-jsonrpc/src/api.rs @@ -23,8 +23,9 @@ use deltachat::ephemeral::Timer; use deltachat::imex; use deltachat::location; use deltachat::message::{ - self, delete_msgs_ex, get_existing_msg_ids, get_msg_read_receipt_count, get_msg_read_receipts, - markseen_msgs, Message, MessageState, MsgId, Viewtype, + self, delete_msgs_ex, dont_truncate_long_messages, get_existing_msg_ids, + get_msg_read_receipt_count, get_msg_read_receipts, markseen_msgs, Message, MessageState, MsgId, + Viewtype, }; use deltachat::peer_channels::{ leave_webxdc_realtime, send_webxdc_realtime_advertisement, send_webxdc_realtime_data, @@ -1373,6 +1374,15 @@ impl CommandApi { MsgId::new(message_id).get_html(&ctx).await } + /// Out out of truncating long messages when loading. + /// + /// Should be used by the UIs that can handle long text messages. + async fn dont_truncate_long_messages(&self, account_id: u32) -> Result<()> { + let ctx = self.get_context(account_id).await?; + dont_truncate_long_messages(&ctx); + Ok(()) + } + /// get multiple messages in one call, /// if loading one message fails the error is stored in the result object in it's place. /// diff --git a/src/chat.rs b/src/chat.rs index 1579dd0d8c..c33070d456 100644 --- a/src/chat.rs +++ b/src/chat.rs @@ -49,7 +49,7 @@ use crate::sync::{self, Sync::*, SyncData}; use crate::tools::{ IsNoneOrEmpty, SystemTime, buf_compress, create_broadcast_secret, create_id, create_outgoing_rfc724_mid, create_smeared_timestamp, create_smeared_timestamps, get_abs_path, - gm2local_offset, normalize_text, smeared_time, time, truncate_msg_text, + gm2local_offset, normalize_text, smeared_time, time, }; use crate::webxdc::StatusUpdateSerial; @@ -1884,7 +1884,8 @@ impl Chat { EphemeralTimer::Enabled { duration } => time().saturating_add(duration.into()), }; - let (msg_text, was_truncated) = truncate_msg_text(context, msg.text.clone()).await?; + let msg_text = msg.text.clone(); + let new_mime_headers = if msg.has_html() { if msg.param.exists(Param::Forwarded) { msg.get_id().get_html(context).await? @@ -1901,13 +1902,6 @@ impl Chat { html_part.write_part(cursor).ok(); String::from_utf8_lossy(&buffer).to_string() }); - let new_mime_headers = new_mime_headers.or_else(|| match was_truncated { - // We need to add some headers so that they are stripped before formatting HTML by - // `MsgId::get_html()`, not a part of the actual text. Let's add "Content-Type", it's - // anyway a useful metadata about the stored text. - true => Some("Content-Type: text/plain; charset=utf-8\r\n\r\n".to_string() + &msg.text), - false => None, - }); let new_mime_headers = match new_mime_headers { Some(h) => Some(tokio::task::block_in_place(move || { buf_compress(h.as_bytes()) diff --git a/src/context.rs b/src/context.rs index 77f20765bb..39f00e495f 100644 --- a/src/context.rs +++ b/src/context.rs @@ -226,7 +226,18 @@ impl WeakContext { pub struct InnerContext { /// Blob directory path pub(crate) blobdir: PathBuf, + pub(crate) sql: Sql, + + /// True if long text messages should be truncated + /// and full message HTML added. + /// + /// This should be set by the UIs that cannot handle + /// long messages but can display HTML messages. + /// + /// Ignored for bots, bots never get truncated messages. + pub(crate) truncate_long_messages: AtomicBool, + pub(crate) smeared_timestamp: SmearedTimestamp, /// The global "ongoing" process state. /// @@ -473,6 +484,7 @@ impl Context { blobdir, running_state: RwLock::new(Default::default()), sql: Sql::new(dbfile), + truncate_long_messages: AtomicBool::new(true), smeared_timestamp: SmearedTimestamp::new(), generating_key_mutex: Mutex::new(()), oauth2_mutex: Mutex::new(()), diff --git a/src/html.rs b/src/html.rs index 5ecbba1a31..a622de56f1 100644 --- a/src/html.rs +++ b/src/html.rs @@ -30,7 +30,7 @@ impl Message { /// The corresponding ffi-function is `dc_msg_has_html()`. /// To get the HTML-code of the message, use `MsgId.get_html()`. pub fn has_html(&self) -> bool { - self.mime_modified + self.mime_modified || self.full_text.is_some() } /// Set HTML-part part of a message that is about to be sent. @@ -270,8 +270,19 @@ impl MsgId { Ok((parser, _)) => Ok(Some(parser.html)), } } else { - warn!(context, "get_html: no mime for {}", self); - Ok(None) + let msg = Message::load_from_db(context, self).await?; + if let Some(full_text) = &msg.full_text { + let html = PlainText { + text: full_text.clone(), + flowed: false, + delsp: false, + } + .to_html(); + Ok(Some(html)) + } else { + warn!(context, "get_html: no mime for {}", self); + Ok(None) + } } } } diff --git a/src/message.rs b/src/message.rs index 82ca488b4b..fd9e9c78d3 100644 --- a/src/message.rs +++ b/src/message.rs @@ -4,6 +4,7 @@ use std::collections::BTreeSet; use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::str; +use std::sync::atomic::Ordering; use anyhow::{Context as _, Result, ensure, format_err}; use deltachat_contact_tools::{VcardContact, parse_vcard}; @@ -35,10 +36,9 @@ use crate::reaction::get_msg_reactions; use crate::sql; use crate::summary::Summary; use crate::sync::SyncData; -use crate::tools::create_outgoing_rfc724_mid; use crate::tools::{ - buf_compress, buf_decompress, get_filebytes, get_filemeta, gm2local_offset, read_file, - sanitize_filename, time, timestamp_to_str, + buf_compress, buf_decompress, create_outgoing_rfc724_mid, get_filebytes, get_filemeta, + gm2local_offset, read_file, sanitize_filename, time, timestamp_to_str, truncate_msg_text, }; /// Message ID, including reserved IDs. @@ -431,7 +431,13 @@ pub struct Message { pub(crate) timestamp_rcvd: i64, pub(crate) ephemeral_timer: EphemeralTimer, pub(crate) ephemeral_timestamp: i64, + + /// Message text, possibly truncated if the message is large. pub(crate) text: String, + + /// Full text if the message text is truncated. + pub(crate) full_text: Option, + /// Text that is added to the end of Message.text /// /// Currently used for adding the download information on pre-messages @@ -556,6 +562,7 @@ impl Message { } _ => String::new(), }; + let msg = Message { id: row.get("id")?, rfc724_mid: row.get::<_, String>("rfc724mid")?, @@ -580,6 +587,7 @@ impl Message { original_msg_id: row.get("original_msg_id")?, mime_modified: row.get("mime_modified")?, text, + full_text: None, additional_text: String::new(), subject: row.get("subject")?, param: row.get::<_, String>("param")?.parse().unwrap_or_default(), @@ -597,6 +605,15 @@ impl Message { .with_context(|| format!("failed to load message {id} from the database"))?; if let Some(msg) = &mut msg { + if !msg.mime_modified { + let (truncated_text, was_truncated) = + truncate_msg_text(context, msg.text.clone()).await?; + if was_truncated { + msg.full_text = Some(msg.text.clone()); + msg.text = truncated_text; + } + } + msg.additional_text = Self::get_additional_text(context, msg.download_state, &msg.param).await?; } @@ -2378,5 +2395,22 @@ impl Viewtype { } } +/// Opt out of truncating long messages. +/// +/// After calling this function, long messages +/// will not be truncated during loading. +/// +/// UIs should call this function if they +/// can handle long messages by cutting them +/// and displaying "Show full message" option. +/// +/// Has no effect for bots which never +/// truncate messages when loading. +pub fn dont_truncate_long_messages(context: &Context) { + context + .truncate_long_messages + .store(false, Ordering::Relaxed); +} + #[cfg(test)] mod message_tests; diff --git a/src/mimeparser.rs b/src/mimeparser.rs index 33a02830ce..a1d3ba5644 100644 --- a/src/mimeparser.rs +++ b/src/mimeparser.rs @@ -32,9 +32,7 @@ use crate::message::{self, Message, MsgId, Viewtype, get_vcard_summary, set_msg_ use crate::param::{Param, Params}; use crate::simplify::{SimplifiedText, simplify}; use crate::sync::SyncItems; -use crate::tools::{ - get_filemeta, parse_receive_headers, smeared_time, time, truncate_msg_text, validate_id, -}; +use crate::tools::{get_filemeta, parse_receive_headers, smeared_time, time, validate_id}; use crate::{chatlist_events, location, tools}; /// Public key extracted from `Autocrypt-Gossip` @@ -1454,12 +1452,6 @@ impl MimeMessage { (simplified_txt, top_quote) }; - let (simplified_txt, was_truncated) = - truncate_msg_text(context, simplified_txt).await?; - if was_truncated { - self.is_mime_modified = was_truncated; - } - if !simplified_txt.is_empty() || simplified_quote.is_some() { let mut part = Part { dehtml_failed, diff --git a/src/mimeparser/mimeparser_tests.rs b/src/mimeparser/mimeparser_tests.rs index b40ed5eaad..0565149697 100644 --- a/src/mimeparser/mimeparser_tests.rs +++ b/src/mimeparser/mimeparser_tests.rs @@ -1285,12 +1285,12 @@ async fn test_mime_modified_large_plain() -> Result<()> { { let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?; - assert!(mimemsg.is_mime_modified); - assert!( - mimemsg.parts[0].msg.matches("just repeated").count() - <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len() + assert!(!mimemsg.is_mime_modified); + assert!(mimemsg.parts[0].msg.matches("just repeated").count() == REPEAT_CNT); + assert_eq!( + mimemsg.parts[0].msg.len() + 1, + REPEAT_TXT.len() * REPEAT_CNT ); - assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len()); } for draft in [false, true] { diff --git a/src/receive_imf/receive_imf_tests.rs b/src/receive_imf/receive_imf_tests.rs index 855600396d..3f26145c6d 100644 --- a/src/receive_imf/receive_imf_tests.rs +++ b/src/receive_imf/receive_imf_tests.rs @@ -3584,39 +3584,17 @@ async fn test_big_forwarded_with_big_attachment() -> Result<()> { .starts_with("this text with 42 chars is just repeated.") ); assert!(msg.get_text().ends_with("[...]")); - assert!(!msg.has_html()); - - let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?; - assert_eq!(msg.get_viewtype(), Viewtype::File); assert!(msg.has_html()); let html = msg.id.get_html(t).await?.unwrap(); - let tail = html - .split_once("Hello!") - .unwrap() - .1 - .split_once("From: AAA") - .unwrap() - .1 - .split_once("aaa@example.org") - .unwrap() - .1 - .split_once("To: Alice") - .unwrap() - .1 - .split_once("alice@example.org") - .unwrap() - .1 - .split_once("Subject: Some subject") - .unwrap() - .1 - .split_once("Date: Fri, 2 Jun 2023 12:29:17 +0000") - .unwrap() - .1; assert_eq!( - tail.matches("this text with 42 chars is just repeated.") + html.matches("this text with 42 chars is just repeated.") .count(), 128 ); + + let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?; + assert_eq!(msg.get_viewtype(), Viewtype::File); + assert!(!msg.has_html()); Ok(()) } diff --git a/src/tools.rs b/src/tools.rs index 6ef3406df4..df893285a9 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -9,6 +9,7 @@ use std::mem; use std::ops::{AddAssign, Deref}; use std::path::{Path, PathBuf}; use std::str::from_utf8; +use std::sync::atomic::Ordering; // If a time value doesn't need to be sent to another host, saved to the db or otherwise used across // program restarts, a monotonically nondecreasing clock (`Instant`) should be used. But as // `Instant` may use `libc::clock_gettime(CLOCK_MONOTONIC)`, e.g. on Android, and does not advance @@ -137,7 +138,9 @@ pub(crate) fn truncate_by_lines( /// /// Returns the resulting text and a bool telling whether a truncation was done. pub(crate) async fn truncate_msg_text(context: &Context, text: String) -> Result<(String, bool)> { - if context.get_config_bool(Config::Bot).await? { + if !context.truncate_long_messages.load(Ordering::Relaxed) + || context.get_config_bool(Config::Bot).await? + { return Ok((text, false)); } // Truncate text if it has too many lines