From 833db92ded1b05cf952e79e1ecb7a3c81ca0306e Mon Sep 17 00:00:00 2001 From: mdecimus Date: Tue, 5 Sep 2023 18:23:52 +0200 Subject: [PATCH] Bump to mail-parser 0.9 --- Cargo.lock | 101 ++--- crates/imap/src/op/append.rs | 4 +- crates/imap/src/op/fetch.rs | 84 ++--- crates/imap/src/op/search.rs | 31 +- crates/jmap-proto/src/types/property.rs | 56 +-- crates/jmap-proto/src/types/value.rs | 8 +- crates/jmap/Cargo.toml | 2 +- crates/jmap/src/email/crypto.rs | 12 +- crates/jmap/src/email/get.rs | 4 +- crates/jmap/src/email/headers.rs | 134 +++---- crates/jmap/src/email/import.rs | 4 +- crates/jmap/src/email/index.rs | 367 +++++++++---------- crates/jmap/src/email/ingest.rs | 14 +- crates/jmap/src/email/parse.rs | 4 +- crates/jmap/src/email/query.rs | 104 +++--- crates/jmap/src/email/set.rs | 4 +- crates/jmap/src/email/snippet.rs | 4 +- crates/jmap/src/services/ingest.rs | 4 +- crates/jmap/src/sieve/ingest.rs | 8 +- crates/managesieve/Cargo.toml | 2 +- crates/smtp/Cargo.toml | 2 +- crates/smtp/src/outbound/dane/dnssec.rs | 2 +- crates/smtp/src/reporting/analysis.rs | 18 +- tests/Cargo.toml | 2 +- tests/resources/jmap_mail_get/headers.json | 22 +- tests/resources/jmap_mail_parse/headers.json | 22 +- tests/resources/test_config.toml | 16 +- tests/src/imap/body_structure.rs | 4 +- tests/src/jmap/crypto.rs | 10 +- tests/src/jmap/delivery.rs | 1 + tests/src/jmap/email_get.rs | 30 +- tests/src/jmap/email_query.rs | 7 +- tests/src/smtp/inbound/milter.rs | 4 +- tests/src/smtp/outbound/dane.rs | 2 +- 34 files changed, 513 insertions(+), 580 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 19dff0a2..72365aa5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1297,7 +1297,7 @@ dependencies = [ "smtp-proto", "sqlx", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tracing", "utils", ] @@ -1480,14 +1480,14 @@ dependencies = [ [[package]] name = "enum-as-inner" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9720bba047d567ffc8a3cba48bf19126600e249ab7f128e9233e6376976a116" +checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" dependencies = [ "heck", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.29", ] [[package]] @@ -2143,7 +2143,7 @@ dependencies = [ "hyper 0.14.27", "rustls 0.21.7", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", ] [[package]] @@ -2214,17 +2214,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" -dependencies = [ - "matches", - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "0.4.0" @@ -2253,7 +2242,7 @@ dependencies = [ "rustls-pemfile", "store", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tracing", "utils", ] @@ -2578,7 +2567,7 @@ dependencies = [ "rustls-native-certs", "thiserror", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-stream", "tokio-util", "url", @@ -2694,8 +2683,8 @@ dependencies = [ [[package]] name = "mail-auth" -version = "0.3.2" -source = "git+https://github.com/stalwartlabs/mail-auth#a6cd1d6cc0a79943903e8154eecc29f2de003e2a" +version = "0.3.3" +source = "git+https://github.com/stalwartlabs/mail-auth#949b8fcd91f329b424e22a3f2bbc3040869f3490" dependencies = [ "ahash 0.8.3", "flate2", @@ -2703,7 +2692,7 @@ dependencies = [ "mail-builder", "mail-parser", "parking_lot", - "quick-xml 0.28.2", + "quick-xml 0.30.0", "ring", "rustls-pemfile", "serde", @@ -2722,8 +2711,8 @@ dependencies = [ [[package]] name = "mail-parser" -version = "0.8.2" -source = "git+https://github.com/stalwartlabs/mail-parser#7c08078617ef9b355da445dbf88df64879eb8059" +version = "0.9.0" +source = "git+https://github.com/stalwartlabs/mail-parser#e5a4e65112fd8aa4c527d37b87413d939f1259a1" dependencies = [ "encoding_rs", "serde", @@ -2740,7 +2729,7 @@ dependencies = [ "rustls 0.21.7", "smtp-proto", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "webpki-roots 0.25.2", ] @@ -2781,7 +2770,7 @@ dependencies = [ "sieve-rs", "store", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tracing", "utils", ] @@ -2807,12 +2796,6 @@ dependencies = [ "regex-automata 0.1.10", ] -[[package]] -name = "matches" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" - [[package]] name = "matchit" version = "0.7.2" @@ -3733,9 +3716,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.28.2" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce5e73202a820a31f8a0ee32ada5e21029c81fd9e3ebf668a40832e4219d9d1" +checksum = "eff6510e86862b57b210fd8cbe8ed3f0d7d600b9c2863cd4549a2e033c66e956" dependencies = [ "memchr", ] @@ -3977,7 +3960,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-util", "tower-service", "url", @@ -4552,7 +4535,7 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "sieve-rs" version = "0.3.1" -source = "git+https://github.com/stalwartlabs/sieve#53ff94606cf4a3e03b8820280ef8fae0bfa500ec" +source = "git+https://github.com/stalwartlabs/sieve#f17c085cbefd2c422d49924795d68255a6c7658c" dependencies = [ "ahash 0.8.3", "bincode", @@ -4637,7 +4620,7 @@ dependencies = [ "smtp-proto", "sqlx", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tracing", "utils", "webpki-roots 0.25.2", @@ -5154,7 +5137,7 @@ dependencies = [ "sqlx", "store", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tracing", "tracing-subscriber", "utils", @@ -5294,17 +5277,6 @@ dependencies = [ "syn 2.0.29", ] -[[package]] -name = "tokio-rustls" -version = "0.23.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" -dependencies = [ - "rustls 0.20.9", - "tokio", - "webpki", -] - [[package]] name = "tokio-rustls" version = "0.24.1" @@ -5336,7 +5308,7 @@ dependencies = [ "log", "rustls 0.21.7", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tungstenite 0.19.0", "webpki-roots 0.23.1", ] @@ -5544,9 +5516,9 @@ dependencies = [ [[package]] name = "trust-dns-proto" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f7f83d1e4a0e4358ac54c5c3681e5d7da5efc5a7a632c90bb6d6669ddd9bc26" +checksum = "0dc775440033cb114085f6f2437682b194fa7546466024b1037e82a48a052a69" dependencies = [ "async-trait", "cfg-if", @@ -5555,44 +5527,45 @@ dependencies = [ "futures-channel", "futures-io", "futures-util", - "idna 0.2.3", + "idna", "ipnet", - "lazy_static", + "once_cell", "rand", "ring", - "rustls 0.20.9", + "rustls 0.21.7", "rustls-pemfile", + "rustls-webpki 0.101.4", "smallvec", "thiserror", "tinyvec", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls", "tracing", "url", - "webpki", ] [[package]] name = "trust-dns-resolver" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aff21aa4dcefb0a1afbfac26deb0adc93888c7d295fb63ab273ef276ba2b7cfe" +checksum = "2dff7aed33ef3e8bf2c9966fccdfed93f93d46f432282ea875cd66faabc6ef2f" dependencies = [ "cfg-if", "futures-util", "ipconfig", - "lazy_static", "lru-cache", + "once_cell", "parking_lot", + "rand", "resolv-conf", - "rustls 0.20.9", + "rustls 0.21.7", "smallvec", "thiserror", "tokio", - "tokio-rustls 0.23.4", + "tokio-rustls", "tracing", "trust-dns-proto", - "webpki-roots 0.22.6", + "webpki-roots 0.25.2", ] [[package]] @@ -5745,7 +5718,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", - "idna 0.4.0", + "idna", "percent-encoding", ] @@ -5778,7 +5751,7 @@ dependencies = [ "serde", "smtp-proto", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", "tracing", "tracing-appender", "tracing-opentelemetry", diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index e23316d4..91398096 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -29,7 +29,7 @@ use imap_proto::{ use jmap::email::ingest::IngestEmail; use jmap_proto::types::{acl::Acl, keyword::Keyword, state::StateChange, type_state::TypeState}; -use mail_parser::Message; +use mail_parser::MessageParser; use tokio::io::AsyncRead; use crate::core::{MailboxId, SelectedMailbox, Session, SessionData}; @@ -135,7 +135,7 @@ impl SessionData { .jmap .email_ingest(IngestEmail { raw_message: &message.message, - message: Message::parse(&message.message), + message: MessageParser::new().parse(&message.message), account_id, account_quota, mailbox_ids: vec![mailbox_id], diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 5104cdb5..e29f0132 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -45,7 +45,7 @@ use jmap_proto::{ property::Property, state::StateChange, type_state::TypeState, value::Value, }, }; -use mail_parser::{GetHeader, Message, PartType, RfcHeader}; +use mail_parser::{Address, GetHeader, HeaderName, Message, MessageParser, PartType}; use store::{ query::log::{Change, Query}, write::{assert::HashedValue, BatchBuilder, F_BITMAP, F_VALUE}, @@ -319,7 +319,7 @@ impl SessionData { }; let message = if let Some(raw_message) = &raw_message { - if let Some(message) = Message::parse(raw_message) { + if let Some(message) = MessageParser::new().parse(raw_message) { message.into() } else { tracing::warn!( @@ -671,8 +671,8 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { }; let content_type = part .headers - .rfc(&RfcHeader::ContentType) - .and_then(|ct| ct.as_content_type_ref()); + .header_value(&HeaderName::ContentType) + .and_then(|ct| ct.as_content_type()); let mut body_md5 = None; let mut extension = BodyPartExtension::default(); @@ -695,18 +695,18 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { fields.body_id = part .headers - .rfc(&RfcHeader::ContentId) - .and_then(|id| id.as_text_ref().map(|id| format!("<{}>", id).into())); + .header_value(&HeaderName::ContentId) + .and_then(|id| id.as_text().map(|id| format!("<{}>", id).into())); fields.body_description = part .headers - .rfc(&RfcHeader::ContentDescription) - .and_then(|ct| ct.as_text_ref().map(|ct| ct.into())); + .header_value(&HeaderName::ContentDescription) + .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_encoding = part .headers - .rfc(&RfcHeader::ContentTransferEncoding) - .and_then(|ct| ct.as_text_ref().map(|ct| ct.into())); + .header_value(&HeaderName::ContentTransferEncoding) + .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_size_octets = body.as_ref().map(|b| b.len()).unwrap_or(0); @@ -730,8 +730,10 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { .map(|b| format!("{:x}", md5::compute(b)).into()); } - extension.body_disposition = - part.headers.rfc(&RfcHeader::ContentDisposition).map(|cd| { + extension.body_disposition = part + .headers + .header_value(&HeaderName::ContentDisposition) + .map(|cd| { let cd = cd.content_type(); ( @@ -747,18 +749,18 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { ) }); - extension.body_language = - part.headers - .rfc(&RfcHeader::ContentLanguage) - .and_then(|hv| { - hv.as_text_list() - .map(|list| list.into_iter().map(|item| item.into()).collect()) - }); + extension.body_language = part + .headers + .header_value(&HeaderName::ContentLanguage) + .and_then(|hv| { + hv.as_text_list() + .map(|list| list.into_iter().map(|item| item.into()).collect()) + }); extension.body_location = part .headers - .rfc(&RfcHeader::ContentLocation) - .and_then(|ct| ct.as_text_ref().map(|ct| ct.into())); + .header_value(&HeaderName::ContentLocation) + .and_then(|ct| ct.as_text().map(|ct| ct.into())); } match &part.body { @@ -1045,27 +1047,27 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { date: self.date().cloned(), subject: self.subject().map(|s| s.into()), from: self - .header_values(RfcHeader::From) + .header_values(HeaderName::From) .flat_map(|a| a.as_imap_address()) .collect(), sender: self - .header_values(RfcHeader::Sender) + .header_values(HeaderName::Sender) .flat_map(|a| a.as_imap_address()) .collect(), reply_to: self - .header_values(RfcHeader::ReplyTo) + .header_values(HeaderName::ReplyTo) .flat_map(|a| a.as_imap_address()) .collect(), to: self - .header_values(RfcHeader::To) + .header_values(HeaderName::To) .flat_map(|a| a.as_imap_address()) .collect(), cc: self - .header_values(RfcHeader::Cc) + .header_values(HeaderName::Cc) .flat_map(|a| a.as_imap_address()) .collect(), bcc: self - .header_values(RfcHeader::Bcc) + .header_values(HeaderName::Bcc) .flat_map(|a| a.as_imap_address()) .collect(), in_reply_to: self.in_reply_to().as_text_list().map(|list| { @@ -1109,15 +1111,7 @@ impl AsImapAddress for mail_parser::HeaderValue<'_> { let mut addresses = Vec::new(); match self { - mail_parser::HeaderValue::Address(addr) => { - if let Some(email) = &addr.address { - addresses.push(fetch::Address::Single(fetch::EmailAddress { - name: addr.name.as_ref().map(|n| n.as_ref().into()), - address: email.as_ref().into(), - })); - } - } - mail_parser::HeaderValue::AddressList(list) => { + mail_parser::HeaderValue::Address(Address::List(list)) => { for addr in list { if let Some(email) = &addr.address { addresses.push(fetch::Address::Single(fetch::EmailAddress { @@ -1127,23 +1121,7 @@ impl AsImapAddress for mail_parser::HeaderValue<'_> { } } } - mail_parser::HeaderValue::Group(group) => { - addresses.push(fetch::Address::Group(fetch::AddressGroup { - name: group.name.as_ref().map(|n| n.as_ref().into()), - addresses: group - .addresses - .iter() - .filter_map(|addr| { - fetch::EmailAddress { - name: addr.name.as_ref().map(|n| n.as_ref().into()), - address: addr.address.as_ref()?.as_ref().into(), - } - .into() - }) - .collect(), - })); - } - mail_parser::HeaderValue::GroupList(list) => { + mail_parser::HeaderValue::Address(Address::Group(list)) => { for group in list { addresses.push(fetch::Address::Group(fetch::AddressGroup { name: group.name.as_ref().map(|n| n.as_ref().into()), diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index cba7c3bc..fa67d208 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -33,7 +33,7 @@ use imap_proto::{ }; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; -use mail_parser::{HeaderName, RfcHeader}; +use mail_parser::HeaderName; use store::{ fts::{builder::MAX_TOKEN_LENGTH, Language}, query::{self, log::Query, sort::Pagination, ResultSet}, @@ -353,17 +353,22 @@ impl SessionData { Language::None, )); } - search::Filter::Header(header, value) => { - if let Some(HeaderName::Rfc(header_name)) = HeaderName::parse(&header) { + search::Filter::Header(header, value) => match HeaderName::parse(&header) { + Some(HeaderName::Other(_)) | None => { + return Err(StatusResponse::no(format!( + "Querying non-RFC header '{header}' is not allowed.", + ))); + } + Some(header_name) => { let is_id = matches!( header_name, - RfcHeader::MessageId - | RfcHeader::InReplyTo - | RfcHeader::References - | RfcHeader::ResentMessageId + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId ); let tokens = if !value.is_empty() { - let header_num = u8::from(header_name).to_string(); + let header_num = header_name.id().to_string(); value .split_ascii_whitespace() .filter_map(|token| { @@ -386,7 +391,7 @@ impl SessionData { 0 => { filters.push(query::Filter::has_raw_text( Property::Headers, - u8::from(header_name).to_string(), + header_name.id().to_string(), )); } 1 => { @@ -406,12 +411,8 @@ impl SessionData { filters.push(query::Filter::End); } } - } else { - return Err(StatusResponse::no(format!( - "Querying non-RFC header '{header}' is not allowed.", - ))); - }; - } + } + }, search::Filter::Keyword(keyword) => { filters.push(query::Filter::is_in_bitmap( Property::Keywords, diff --git a/crates/jmap-proto/src/types/property.rs b/crates/jmap-proto/src/types/property.rs index 31380408..4e0dbe6b 100644 --- a/crates/jmap-proto/src/types/property.rs +++ b/crates/jmap-proto/src/types/property.rs @@ -23,7 +23,7 @@ use std::fmt::{Display, Formatter}; -use mail_parser::RfcHeader; +use mail_parser::HeaderName; use serde::Serialize; use store::write::{DeserializeFrom, SerializeInto}; @@ -692,19 +692,19 @@ impl Property { } } - pub fn as_rfc_header(&self) -> RfcHeader { + pub fn as_rfc_header(&self) -> HeaderName<'static> { match self { - Property::MessageId => RfcHeader::MessageId, - Property::InReplyTo => RfcHeader::InReplyTo, - Property::References => RfcHeader::References, - Property::Sender => RfcHeader::Sender, - Property::From => RfcHeader::From, - Property::To => RfcHeader::To, - Property::Cc => RfcHeader::Cc, - Property::Bcc => RfcHeader::Bcc, - Property::ReplyTo => RfcHeader::ReplyTo, - Property::Subject => RfcHeader::Subject, - Property::SentAt => RfcHeader::Date, + Property::MessageId => HeaderName::MessageId, + Property::InReplyTo => HeaderName::InReplyTo, + Property::References => HeaderName::References, + Property::Sender => HeaderName::Sender, + Property::From => HeaderName::From, + Property::To => HeaderName::To, + Property::Cc => HeaderName::Cc, + Property::Bcc => HeaderName::Bcc, + Property::ReplyTo => HeaderName::ReplyTo, + Property::Subject => HeaderName::Subject, + Property::SentAt => HeaderName::Date, _ => unreachable!(), } } @@ -994,21 +994,21 @@ impl From for u8 { } } -impl From for Property { - fn from(value: RfcHeader) -> Self { - match value { - RfcHeader::Subject => Property::Subject, - RfcHeader::From => Property::From, - RfcHeader::To => Property::To, - RfcHeader::Cc => Property::Cc, - RfcHeader::Date => Property::SentAt, - RfcHeader::Bcc => Property::Bcc, - RfcHeader::ReplyTo => Property::ReplyTo, - RfcHeader::Sender => Property::Sender, - RfcHeader::InReplyTo => Property::InReplyTo, - RfcHeader::MessageId => Property::MessageId, - RfcHeader::References => Property::References, - RfcHeader::ResentMessageId => Property::EmailIds, +impl Property { + pub fn from_header(header: &HeaderName) -> Self { + match header { + HeaderName::Subject => Property::Subject, + HeaderName::From => Property::From, + HeaderName::To => Property::To, + HeaderName::Cc => Property::Cc, + HeaderName::Date => Property::SentAt, + HeaderName::Bcc => Property::Bcc, + HeaderName::ReplyTo => Property::ReplyTo, + HeaderName::Sender => Property::Sender, + HeaderName::InReplyTo => Property::InReplyTo, + HeaderName::MessageId => Property::MessageId, + HeaderName::References => Property::References, + HeaderName::ResentMessageId => Property::EmailIds, _ => unreachable!(), } } diff --git a/crates/jmap-proto/src/types/value.rs b/crates/jmap-proto/src/types/value.rs index d6a68506..62d00527 100644 --- a/crates/jmap-proto/src/types/value.rs +++ b/crates/jmap-proto/src/types/value.rs @@ -496,13 +496,7 @@ impl From> for Value { group .addresses .into_iter() - .filter_map(|addr| { - if addr.address.as_ref()?.contains('@') { - Some(addr.into()) - } else { - None - } - }) + .map(Value::from) .collect::>(), ), ), diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 893d42bf..c1e0e8b8 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -14,7 +14,7 @@ smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "serde_support", "ludicrous_mode"] } mail-builder = { git = "https://github.com/stalwartlabs/mail-builder", features = ["ludicrous_mode"] } mail-send = { git = "https://github.com/stalwartlabs/mail-send", default-features = false, features = ["cram-md5", "skip-ehlo"] } -sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } +sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } serde = { version = "1.0", features = ["derive"]} serde_json = "1.0" hyper = { version = "1.0.0-rc.4", features = ["server", "http1", "http2"] } diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index 9e06ce78..194d8ece 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -26,7 +26,7 @@ use std::{borrow::Cow, collections::BTreeSet, fmt::Display}; use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit}; use jmap_proto::types::{collection::Collection, property::Property}; use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary}; -use mail_parser::{decoders::base64::base64_decode, Message, MimeHeaders}; +use mail_parser::{decoders::base64::base64_decode, Message, MessageParser, MimeHeaders}; use pgp::{composed, crypto::sym::SymmetricKeyAlgorithm, Deserializable, SignedPublicKey}; use rand::{rngs::StdRng, RngCore, SeedableRng}; use rasn::types::{ObjectIdentifier, OctetString}; @@ -632,11 +632,11 @@ impl JMAP { }; // Try a test encryption - if let Err(EncryptMessageError::Error(message)) = - Message::parse("Subject: test\r\ntest\r\n".as_bytes()) - .unwrap() - .encrypt(¶ms) - .await + if let Err(EncryptMessageError::Error(message)) = MessageParser::new() + .parse("Subject: test\r\ntest\r\n".as_bytes()) + .unwrap() + .encrypt(¶ms) + .await { return Err(Cow::from(message)); } diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 0ecfc500..aa3f70bf 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -30,7 +30,7 @@ use jmap_proto::{ property::Property, value::Value, }, }; -use mail_parser::Message; +use mail_parser::MessageParser; use crate::{auth::AccessToken, email::headers::HeaderToValue, JMAP}; @@ -192,7 +192,7 @@ impl JMAP { vec![] }; let message = if !raw_message.is_empty() { - let message = Message::parse(&raw_message); + let message = MessageParser::new().parse(&raw_message); if message.is_none() { tracing::warn!( event = "parse-error", diff --git a/crates/jmap/src/email/headers.rs b/crates/jmap/src/email/headers.rs index a7e90d31..23de70d1 100644 --- a/crates/jmap/src/email/headers.rs +++ b/crates/jmap/src/email/headers.rs @@ -41,7 +41,7 @@ use mail_builder::{ }, MessageBuilder, }; -use mail_parser::{parsers::MessageStream, Addr, HeaderName, HeaderValue, MessagePart, RfcHeader}; +use mail_parser::{parsers::MessageStream, Addr, HeaderName, HeaderValue, MessagePart}; pub trait IntoForm { fn into_form(self, form: &HeaderForm) -> Value; @@ -71,52 +71,24 @@ impl HeaderToValue for MessagePart<'_> { header.form, header.all, ), - Property::Sender => ( - HeaderName::Rfc(RfcHeader::Sender), - HeaderForm::Addresses, - false, - ), - Property::From => ( - HeaderName::Rfc(RfcHeader::From), - HeaderForm::Addresses, - false, - ), - Property::To => (HeaderName::Rfc(RfcHeader::To), HeaderForm::Addresses, false), - Property::Cc => (HeaderName::Rfc(RfcHeader::Cc), HeaderForm::Addresses, false), - Property::Bcc => ( - HeaderName::Rfc(RfcHeader::Bcc), - HeaderForm::Addresses, - false, - ), - Property::ReplyTo => ( - HeaderName::Rfc(RfcHeader::ReplyTo), - HeaderForm::Addresses, - false, - ), - Property::Subject => (HeaderName::Rfc(RfcHeader::Subject), HeaderForm::Text, false), - Property::MessageId => ( - HeaderName::Rfc(RfcHeader::MessageId), - HeaderForm::MessageIds, - false, - ), - Property::InReplyTo => ( - HeaderName::Rfc(RfcHeader::InReplyTo), - HeaderForm::MessageIds, - false, - ), - Property::References => ( - HeaderName::Rfc(RfcHeader::References), - HeaderForm::MessageIds, - false, - ), - Property::SentAt => (HeaderName::Rfc(RfcHeader::Date), HeaderForm::Date, false), + Property::Sender => (HeaderName::Sender, HeaderForm::Addresses, false), + Property::From => (HeaderName::From, HeaderForm::Addresses, false), + Property::To => (HeaderName::To, HeaderForm::Addresses, false), + Property::Cc => (HeaderName::Cc, HeaderForm::Addresses, false), + Property::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false), + Property::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false), + Property::Subject => (HeaderName::Subject, HeaderForm::Text, false), + Property::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false), + Property::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false), + Property::References => (HeaderName::References, HeaderForm::MessageIds, false), + Property::SentAt => (HeaderName::Date, HeaderForm::Date, false), _ => return Value::Null, }; let mut headers = Vec::new(); match (&header_name, &form) { - (HeaderName::Other(_), _) | (HeaderName::Rfc(_), HeaderForm::Raw) => { + (HeaderName::Other(_), _) | (_, HeaderForm::Raw) => { let header_name = header_name.as_str(); for header in self.headers().iter().rev() { if header.name.as_str().eq_ignore_ascii_case(header_name) { @@ -142,7 +114,7 @@ impl HeaderToValue for MessagePart<'_> { } } } - (HeaderName::Rfc(header_name), _) => { + (header_name, _) => { let header_name = header_name.as_str(); for header in self.headers().iter().rev() { if header.name.as_str().eq_ignore_ascii_case(header_name) { @@ -197,58 +169,44 @@ impl IntoForm for HeaderValue<'_> { (HeaderValue::Text(text), HeaderForm::MessageIds) => Value::List(vec![text.into()]), (HeaderValue::TextList(texts), HeaderForm::MessageIds) => texts.into(), (HeaderValue::DateTime(datetime), HeaderForm::Date) => datetime.into(), + (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::URLs) => { + Value::List( + addrlist + .into_iter() + .filter_map(|addr| match addr { + Addr { + address: Some(addr), + .. + } if addr.contains(':') => Some(addr.into()), + _ => None, + }) + .collect(), + ) + } + (HeaderValue::Address(mail_parser::Address::List(addrlist)), HeaderForm::Addresses) => { + addrlist.into() + } ( - HeaderValue::Address(Addr { - address: Some(addr), - .. - }), - HeaderForm::URLs, - ) if addr.contains(':') => Value::List(vec![addr.into()]), - (HeaderValue::AddressList(addrlist), HeaderForm::URLs) => Value::List( - addrlist - .into_iter() - .filter_map(|addr| match addr { - Addr { - address: Some(addr), - .. - } if addr.contains(':') => Some(addr.into()), - _ => None, - }) - .collect(), - ), - (HeaderValue::Address(addr), HeaderForm::Addresses) => Value::List(vec![addr.into()]), - (HeaderValue::AddressList(addrlist), HeaderForm::Addresses) => addrlist.into(), - (HeaderValue::Group(group), HeaderForm::Addresses) => group.addresses.into(), - (HeaderValue::GroupList(grouplist), HeaderForm::Addresses) => Value::List( + HeaderValue::Address(mail_parser::Address::Group(grouplist)), + HeaderForm::Addresses, + ) => Value::List( grouplist .into_iter() .flat_map(|group| group.addresses) - .filter_map(|addr| { - if addr.address.as_ref()?.contains('@') { - Some(addr.into()) - } else { - None - } - }) + .map(Value::from) .collect(), ), - (HeaderValue::Address(addr), HeaderForm::GroupedAddresses) => { - Value::List(vec![Object::with_capacity(2) - .with_property(Property::Name, Value::Null) - .with_property(Property::Addresses, Value::List(vec![addr.into()])) - .into()]) - } - - (HeaderValue::AddressList(addrlist), HeaderForm::GroupedAddresses) => { - Value::List(vec![Object::with_capacity(2) - .with_property(Property::Name, Value::Null) - .with_property(Property::Addresses, addrlist) - .into()]) - } - (HeaderValue::Group(group), HeaderForm::GroupedAddresses) => { - Value::List(vec![group.into()]) - } - (HeaderValue::GroupList(grouplist), HeaderForm::GroupedAddresses) => grouplist.into(), + ( + HeaderValue::Address(mail_parser::Address::List(addrlist)), + HeaderForm::GroupedAddresses, + ) => Value::List(vec![Object::with_capacity(2) + .with_property(Property::Name, Value::Null) + .with_property(Property::Addresses, addrlist) + .into()]), + ( + HeaderValue::Address(mail_parser::Address::Group(grouplist)), + HeaderForm::GroupedAddresses, + ) => grouplist.into(), _ => Value::Null, } diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 06ec02d6..9c598763 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -36,7 +36,7 @@ use jmap_proto::{ type_state::TypeState, }, }; -use mail_parser::Message; +use mail_parser::MessageParser; use utils::map::vec_map::VecMap; use crate::{auth::AccessToken, IngestError, JMAP}; @@ -134,7 +134,7 @@ impl JMAP { match self .email_ingest(IngestEmail { raw_message: &raw_message, - message: Message::parse(&raw_message), + message: MessageParser::new().parse(&raw_message), account_id, account_quota, mailbox_ids, diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index 04cafe84..f5170f1d 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -35,7 +35,7 @@ use jmap_proto::{ use mail_parser::{ decoders::html::html_to_text, parsers::{fields::thread::thread_name, preview::preview_text}, - Addr, GetHeader, Group, HeaderName, HeaderValue, Message, MessagePart, PartType, RfcHeader, + Addr, Address, GetHeader, Group, HeaderName, HeaderValue, Message, MessagePart, PartType, }; use store::{ fts::{ @@ -53,6 +53,7 @@ pub const MAX_SORT_FIELD_LENGTH: usize = 255; pub const MAX_STORED_FIELD_LENGTH: usize = 512; pub const PREVIEW_LENGTH: usize = 256; +#[derive(Debug)] pub struct SortedAddressBuilder { last_is_space: bool, pub buf: String, @@ -120,176 +121,177 @@ impl IndexMessage for BatchBuilder { language = part_language; let mut extra_ids = Vec::new(); for header in part.headers.into_iter().rev() { - if let HeaderName::Rfc(rfc_header) = header.name { - // Index hasHeader property - let header_num = (rfc_header as u8).to_string(); - fts.index_raw_token(Property::Headers, &header_num); + if matches!(header.name, HeaderName::Other(_)) { + continue; + } + // Index hasHeader property + let header_num = header.name.id().to_string(); + fts.index_raw_token(Property::Headers, &header_num); - match rfc_header { - RfcHeader::MessageId - | RfcHeader::InReplyTo - | RfcHeader::References - | RfcHeader::ResentMessageId => { - header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - self.value(Property::MessageId, id, F_INDEX); - } + match header.name { + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + self.value(Property::MessageId, id, F_INDEX); + } - // Index ids without stemming - if id.len() < MAX_TOKEN_LENGTH { - fts.index_raw_token( - Property::Headers, - format!("{header_num}{id}"), - ); - } - }); - - if matches!( - rfc_header, - RfcHeader::MessageId - | RfcHeader::InReplyTo - | RfcHeader::References - ) && !seen_headers[rfc_header as usize] - { - metadata.append( - rfc_header.into(), - header - .value - .trim_text(MAX_STORED_FIELD_LENGTH) - .into_form(&HeaderForm::MessageIds), + // Index ids without stemming + if id.len() < MAX_TOKEN_LENGTH { + fts.index_raw_token( + Property::Headers, + format!("{header_num}{id}"), ); - seen_headers[rfc_header as usize] = true; - } else { - header.value.into_visit_text(|id| { - extra_ids.push(Value::Text(id)); - }); } + }); + + if matches!( + header.name, + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + ) && !seen_headers[header.name.id() as usize] + { + metadata.append( + Property::from_header(&header.name), + header + .value + .trim_text(MAX_STORED_FIELD_LENGTH) + .into_form(&HeaderForm::MessageIds), + ); + seen_headers[header.name.id() as usize] = true; + } else { + header.value.into_visit_text(|id| { + extra_ids.push(Value::Text(id)); + }); } - RfcHeader::From - | RfcHeader::To - | RfcHeader::Cc - | RfcHeader::Bcc - | RfcHeader::ReplyTo - | RfcHeader::Sender => { - let property = Property::from(rfc_header); - let seen_header = seen_headers[rfc_header as usize]; - if matches!( - rfc_header, - RfcHeader::From - | RfcHeader::To - | RfcHeader::Cc - | RfcHeader::Bcc - ) { - let mut sort_text = SortedAddressBuilder::new(); - let mut found_addr = seen_header; + } + HeaderName::From + | HeaderName::To + | HeaderName::Cc + | HeaderName::Bcc + | HeaderName::ReplyTo + | HeaderName::Sender => { + let property = Property::from_header(&header.name); + let seen_header = seen_headers[header.name.id() as usize]; + if matches!( + header.name, + HeaderName::From + | HeaderName::To + | HeaderName::Cc + | HeaderName::Bcc + ) { + let mut sort_text = SortedAddressBuilder::new(); + let mut found_addr = seen_header; - header.value.visit_addresses(|element, value| { - if !found_addr { - match element { - AddressElement::Name => { - found_addr = !sort_text.push(value); - } - AddressElement::Address => { - sort_text.push(value); - found_addr = true; - } - AddressElement::GroupName => (), + header.value.visit_addresses(|element, value| { + if !found_addr { + match element { + AddressElement::Name => { + found_addr = !sort_text.push(value); } + AddressElement::Address => { + sort_text.push(value); + found_addr = true; + } + AddressElement::GroupName => (), } - - // Index an address name or email without stemming - fts.index_raw(u8::from(&property), value); - }); - - if !seen_header { - // Add address to inverted index - self.value(u8::from(&property), sort_text.build(), F_INDEX); } - } + + // Index an address name or email without stemming + fts.index_raw(u8::from(&property), value); + }); if !seen_header { - // Add address to metadata - metadata.append( - property, - header - .value - .trim_text(MAX_STORED_FIELD_LENGTH) - .into_form(&HeaderForm::Addresses), - ); - seen_headers[rfc_header as usize] = true; + // Add address to inverted index + self.value(u8::from(&property), sort_text.build(), F_INDEX); } } - RfcHeader::Date => { - if !seen_headers[rfc_header as usize] { - if let HeaderValue::DateTime(datetime) = &header.value { - self.value( - Property::SentAt, - datetime.to_timestamp() as u64, - F_INDEX, - ); - } - metadata.append( - Property::SentAt, - header.value.into_form(&HeaderForm::Date), - ); - seen_headers[rfc_header as usize] = true; - } + + if !seen_header { + // Add address to metadata + metadata.append( + property, + header + .value + .trim_text(MAX_STORED_FIELD_LENGTH) + .into_form(&HeaderForm::Addresses), + ); + seen_headers[header.name.id() as usize] = true; } - RfcHeader::Subject => { - // Index subject - let subject = match &header.value { - HeaderValue::Text(text) => text.clone(), - HeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().clone() - } - _ => "".into(), - }; - - if !seen_headers[rfc_header as usize] { - // Add to metadata - metadata.append( - Property::Subject, - header - .value - .trim_text(MAX_STORED_FIELD_LENGTH) - .into_form(&HeaderForm::Text), - ); - - // Index thread name - let thread_name = thread_name(&subject); + } + HeaderName::Date => { + if !seen_headers[header.name.id() as usize] { + if let HeaderValue::DateTime(datetime) = &header.value { self.value( - Property::Subject, - if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" - }, + Property::SentAt, + datetime.to_timestamp() as u64, F_INDEX, ); - - seen_headers[rfc_header as usize] = true; } - - // Index subject for FTS - fts.index(Property::Subject, subject, language); + metadata.append( + Property::SentAt, + header.value.into_form(&HeaderForm::Date), + ); + seen_headers[header.name.id() as usize] = true; } - - RfcHeader::Comments | RfcHeader::Keywords | RfcHeader::ListId => { - // Index headers - header.value.visit_text(|text| { - for token in text.split_ascii_whitespace() { - if token.len() < MAX_TOKEN_LENGTH { - fts.index_raw_token( - Property::Headers, - format!("{header_num}{}", token.to_lowercase()), - ); - } - } - }); - } - _ => (), } + HeaderName::Subject => { + // Index subject + let subject = match &header.value { + HeaderValue::Text(text) => text.clone(), + HeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().clone() + } + _ => "".into(), + }; + + if !seen_headers[header.name.id() as usize] { + // Add to metadata + metadata.append( + Property::Subject, + header + .value + .trim_text(MAX_STORED_FIELD_LENGTH) + .into_form(&HeaderForm::Text), + ); + + // Index thread name + let thread_name = thread_name(&subject); + self.value( + Property::Subject, + if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + }, + F_INDEX, + ); + + seen_headers[header.name.id() as usize] = true; + } + + // Index subject for FTS + fts.index(Property::Subject, subject, language); + } + + HeaderName::Comments | HeaderName::Keywords | HeaderName::ListId => { + // Index headers + header.value.visit_text(|text| { + for token in text.split_ascii_whitespace() { + if token.len() < MAX_TOKEN_LENGTH { + fts.index_raw_token( + Property::Headers, + format!("{header_num}{}", token.to_lowercase()), + ); + } + } + }); + } + _ => (), } } @@ -300,7 +302,7 @@ impl IndexMessage for BatchBuilder { } // Add subject to index if missing - if !seen_headers[RfcHeader::Subject as usize] { + if !seen_headers[HeaderName::Subject.id() as usize] { self.value(Property::Subject, "!", F_INDEX); } @@ -347,7 +349,7 @@ impl IndexMessage for BatchBuilder { .language() .unwrap_or(Language::Unknown); if let Some(HeaderValue::Text(subject)) = - nested_message.remove_header_rfc(RfcHeader::Subject) + nested_message.remove_header(HeaderName::Subject) { fts.index( Property::Attachments, @@ -549,17 +551,19 @@ trait GetContentLanguage { impl GetContentLanguage for MessagePart<'_> { fn language(&self) -> Option { - self.headers.rfc(&RfcHeader::ContentLanguage).and_then(|v| { - Language::from_iso_639(match v { - HeaderValue::Text(v) => v.as_ref(), - HeaderValue::TextList(v) => v.first()?, - _ => { - return None; - } + self.headers + .header_value(&HeaderName::ContentLanguage) + .and_then(|v| { + Language::from_iso_639(match v { + HeaderValue::Text(v) => v.as_ref(), + HeaderValue::TextList(v) => v.first()?, + _ => { + return None; + } + }) + .unwrap_or(Language::Unknown) + .into() }) - .unwrap_or(Language::Unknown) - .into() - }) } } @@ -569,6 +573,7 @@ trait VisitValues { fn into_visit_text(self, visitor: impl FnMut(String)); } +#[derive(Debug, PartialEq, Eq)] enum AddressElement { Name, Address, @@ -578,15 +583,7 @@ enum AddressElement { impl VisitValues for HeaderValue<'_> { fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { match self { - HeaderValue::Address(addr) => { - if let Some(name) = &addr.name { - visitor(AddressElement::Name, name); - } - if let Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); - } - } - HeaderValue::AddressList(addr_list) => { + HeaderValue::Address(Address::List(addr_list)) => { for addr in addr_list { if let Some(name) = &addr.name { visitor(AddressElement::Name, name); @@ -596,21 +593,7 @@ impl VisitValues for HeaderValue<'_> { } } } - HeaderValue::Group(group) => { - if let Some(name) = &group.name { - visitor(AddressElement::GroupName, name); - } - - for addr in &group.addresses { - if let Some(name) = &addr.name { - visitor(AddressElement::Name, name); - } - if let Some(addr) = &addr.address { - visitor(AddressElement::Address, addr); - } - } - } - HeaderValue::GroupList(groups) => { + HeaderValue::Address(Address::Group(groups)) => { for group in groups { if let Some(name) = &group.name { visitor(AddressElement::GroupName, name); @@ -666,10 +649,12 @@ pub trait TrimTextValue { impl TrimTextValue for HeaderValue<'_> { fn trim_text(self, length: usize) -> Self { match self { - HeaderValue::Address(v) => HeaderValue::Address(v.trim_text(length)), - HeaderValue::AddressList(v) => HeaderValue::AddressList(v.trim_text(length)), - HeaderValue::Group(v) => HeaderValue::Group(v.trim_text(length)), - HeaderValue::GroupList(v) => HeaderValue::GroupList(v.trim_text(length)), + HeaderValue::Address(Address::List(v)) => { + HeaderValue::Address(Address::List(v.trim_text(length))) + } + HeaderValue::Address(Address::Group(v)) => { + HeaderValue::Address(Address::Group(v.trim_text(length))) + } HeaderValue::Text(v) => HeaderValue::Text(v.trim_text(length)), HeaderValue::TextList(v) => HeaderValue::TextList(v.trim_text(length)), v => v, diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index a54c66b9..34add63a 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -31,7 +31,7 @@ use jmap_proto::{ }, }; use mail_parser::{ - parsers::fields::thread::thread_name, HeaderName, HeaderValue, Message, PartType, RfcHeader, + parsers::fields::thread::thread_name, HeaderName, HeaderValue, Message, PartType, }; use store::{ ahash::AHashSet, @@ -103,12 +103,10 @@ impl JMAP { let mut subject = ""; for header in message.root_part().headers().iter().rev() { match header.name { - HeaderName::Rfc( - RfcHeader::MessageId - | RfcHeader::InReplyTo - | RfcHeader::References - | RfcHeader::ResentMessageId, - ) => match &header.value { + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId => match &header.value { HeaderValue::Text(id) if id.len() < MAX_ID_LENGTH => { references.push(id.as_ref()); } @@ -121,7 +119,7 @@ impl JMAP { } _ => (), }, - HeaderName::Rfc(RfcHeader::Subject) if subject.is_empty() => { + HeaderName::Subject if subject.is_empty() => { subject = thread_name(match &header.value { HeaderValue::Text(text) => text.as_ref(), HeaderValue::TextList(list) if !list.is_empty() => { diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index 691b8eaa..1d6370a3 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -28,7 +28,7 @@ use jmap_proto::{ types::{property::Property, value::Value}, }; use mail_parser::{ - decoders::html::html_to_text, parsers::preview::preview_text, Message, PartType, + decoders::html::html_to_text, parsers::preview::preview_text, MessageParser, PartType, }; use utils::map::vec_map::VecMap; @@ -108,7 +108,7 @@ impl JMAP { continue; } }; - let message = if let Some(message) = Message::parse(&raw_message) { + let message = if let Some(message) = MessageParser::new().parse(&raw_message) { message } else { response.not_parsable.push(blob_id); diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 684bbaa1..ac3b57e4 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -27,7 +27,7 @@ use jmap_proto::{ object::email::QueryArguments, types::{acl::Acl, collection::Collection, keyword::Keyword, property::Property}, }; -use mail_parser::{HeaderName, RfcHeader}; +use mail_parser::HeaderName; use store::{ fts::{builder::MAX_TOKEN_LENGTH, Language}, query::{self}, @@ -158,63 +158,67 @@ impl JMAP { let header_name = header.next().ok_or_else(|| { MethodError::InvalidArguments("Header name is missing.".to_string()) })?; - if let Some(HeaderName::Rfc(header_name)) = HeaderName::parse(&header_name) { - let is_id = matches!( - header_name, - RfcHeader::MessageId - | RfcHeader::InReplyTo - | RfcHeader::References - | RfcHeader::ResentMessageId - ); - let tokens = if let Some(header_value) = header.next() { - let header_num = u8::from(header_name).to_string(); - header_value - .split_ascii_whitespace() - .filter_map(|token| { - if token.len() < MAX_TOKEN_LENGTH { - if is_id { - format!("{header_num}{token}") + + match HeaderName::parse(&header_name) { + Some(HeaderName::Other(_)) | None => { + return Err(MethodError::InvalidArguments(format!( + "Querying non-RFC header '{header_name}' is not allowed.", + ))); + } + Some(header_name) => { + let is_id = matches!( + header_name, + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId + ); + let tokens = if let Some(header_value) = header.next() { + let header_num = header_name.id().to_string(); + header_value + .split_ascii_whitespace() + .filter_map(|token| { + if token.len() < MAX_TOKEN_LENGTH { + if is_id { + format!("{header_num}{token}") + } else { + format!("{header_num}{}", token.to_lowercase()) + } + .into() } else { - format!("{header_num}{}", token.to_lowercase()) + None } - .into() - } else { - None - } - }) - .collect::>() - } else { - vec![] - }; - match tokens.len() { - 0 => { - filters.push(query::Filter::has_raw_text( - Property::Headers, - u8::from(header_name).to_string(), - )); - } - 1 => { - filters.push(query::Filter::has_raw_text( - Property::Headers, - tokens.into_iter().next().unwrap(), - )); - } - _ => { - filters.push(query::Filter::And); - for token in tokens { + }) + .collect::>() + } else { + vec![] + }; + match tokens.len() { + 0 => { filters.push(query::Filter::has_raw_text( Property::Headers, - token, + header_name.id().to_string(), )); } - filters.push(query::Filter::End); + 1 => { + filters.push(query::Filter::has_raw_text( + Property::Headers, + tokens.into_iter().next().unwrap(), + )); + } + _ => { + filters.push(query::Filter::And); + for token in tokens { + filters.push(query::Filter::has_raw_text( + Property::Headers, + token, + )); + } + filters.push(query::Filter::End); + } } } - } else { - return Err(MethodError::InvalidArguments(format!( - "Querying non-RFC header '{header_name}' is not allowed.", - ))); - }; + } } // Non-standard diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index ea5b3ce1..a24a9e0c 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -50,7 +50,7 @@ use mail_builder::{ mime::{BodyPart, MimePart}, MessageBuilder, }; -use mail_parser::Message; +use mail_parser::MessageParser; use store::{ ahash::AHashSet, fts::term_index::TokenIndex, @@ -730,7 +730,7 @@ impl JMAP { match self .email_ingest(IngestEmail { raw_message: &raw_message, - message: Message::parse(&raw_message), + message: MessageParser::new().parse(&raw_message), account_id, account_quota, mailbox_ids: mailboxes, diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 05e2653e..0931b82c 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -29,7 +29,7 @@ use jmap_proto::{ }, types::{acl::Acl, collection::Collection}, }; -use mail_parser::{decoders::html::html_to_text, Message, PartType}; +use mail_parser::{decoders::html::html_to_text, MessageParser, PartType}; use store::{ fts::{ builder::MAX_TOKEN_LENGTH, @@ -150,7 +150,7 @@ impl JMAP { }; // Parse message - let message = if let Some(message) = Message::parse(&raw_message) { + let message = if let Some(message) = MessageParser::new().parse(&raw_message) { message } else { response.not_found.push(email_id); diff --git a/crates/jmap/src/services/ingest.rs b/crates/jmap/src/services/ingest.rs index e52b548c..566e4821 100644 --- a/crates/jmap/src/services/ingest.rs +++ b/crates/jmap/src/services/ingest.rs @@ -22,7 +22,7 @@ */ use jmap_proto::types::{state::StateChange, type_state::TypeState}; -use mail_parser::Message; +use mail_parser::MessageParser; use store::ahash::AHashMap; use utils::ipc::{DeliveryResult, IngestMessage}; @@ -97,7 +97,7 @@ impl JMAP { self.email_ingest(IngestEmail { raw_message: &raw_message, - message: Message::parse(&raw_message), + message: MessageParser::new().parse(&raw_message), account_id: uid, account_quota, mailbox_ids: vec![INBOX_ID], diff --git a/crates/jmap/src/sieve/ingest.rs b/crates/jmap/src/sieve/ingest.rs index 97e4d53c..b47344ac 100644 --- a/crates/jmap/src/sieve/ingest.rs +++ b/crates/jmap/src/sieve/ingest.rs @@ -24,7 +24,7 @@ use std::borrow::Cow; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; -use mail_parser::Message; +use mail_parser::MessageParser; use sieve::{Envelope, Event, Input, Mailbox, Recipient}; use smtp::core::{NullIo, Session, SessionAddress}; use store::{ @@ -59,7 +59,7 @@ impl JMAP { mut active_script: ActiveScript, ) -> Result { // Parse message - let message = if let Some(message) = Message::parse(raw_message) { + let message = if let Some(message) = MessageParser::new().parse(raw_message) { message } else { return Err(IngestError::Permanent { @@ -428,7 +428,9 @@ impl JMAP { // Parse message if needed let message = if message_id == 0 && !instance.has_message_changed() { instance.take_message() - } else if let Some(message) = Message::parse(&sieve_message.raw_message) { + } else if let Some(message) = + MessageParser::new().parse(sieve_message.raw_message.as_ref()) + { message } else { tracing::error!( diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index fc15c133..d76cedce 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -14,7 +14,7 @@ store = { path = "../store" } utils = { path = "../utils" } mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "ludicrous_mode"] } mail-send = { git = "https://github.com/stalwartlabs/mail-send", default-features = false, features = ["cram-md5", "skip-ehlo"] } -sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } +sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } rustls = "0.21.0" rustls-pemfile = "1.0" tokio = { version = "1.23", features = ["full"] } diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index aedecca4..d01a5702 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -19,7 +19,7 @@ mail-send = { git = "https://github.com/stalwartlabs/mail-send", default-feature mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "ludicrous_mode"] } mail-builder = { git = "https://github.com/stalwartlabs/mail-builder", features = ["ludicrous_mode"] } smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } -sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } +sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } ahash = { version = "0.8" } rustls = "0.21.0" rustls-pemfile = "1.0" diff --git a/crates/smtp/src/outbound/dane/dnssec.rs b/crates/smtp/src/outbound/dane/dnssec.rs index 8372b982..2cb1cd09 100644 --- a/crates/smtp/src/outbound/dane/dnssec.rs +++ b/crates/smtp/src/outbound/dane/dnssec.rs @@ -45,7 +45,7 @@ impl DnssecResolver { options: ResolverOpts, ) -> Result { Ok(Self { - resolver: AsyncResolver::tokio(config, options)?, + resolver: AsyncResolver::tokio(config, options), }) } } diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index 205b0968..f1adb122 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -35,7 +35,7 @@ use mail_auth::{ report::{tlsrpt::TlsReport, ActionDisposition, DmarcResult, Feedback, Report}, zip, }; -use mail_parser::{DateTime, HeaderValue, Message, MimeHeaders, PartType}; +use mail_parser::{DateTime, MessageParser, MimeHeaders, PartType}; use crate::core::SMTP; @@ -65,21 +65,17 @@ impl AnalyzeReport for Arc { fn analyze_report(&self, message: Arc>) { let core = self.clone(); self.worker_pool.spawn(move || { - let message = if let Some(message) = Message::parse(&message) { + let message = if let Some(message) = MessageParser::default().parse(message.as_ref()) { message } else { tracing::debug!(context = "report", "Failed to parse message."); return; }; - let from = match message.from() { - HeaderValue::Address(addr) => addr.address.as_ref().map(|a| a.as_ref()), - HeaderValue::AddressList(addr_list) => addr_list - .last() - .and_then(|a| a.address.as_ref()) - .map(|a| a.as_ref()), - _ => None, - } - .unwrap_or("unknown"); + let from = message + .from() + .and_then(|a| a.last()) + .and_then(|a| a.address()) + .unwrap_or("unknown"); let mut reports = Vec::new(); for part in &message.parts { diff --git a/tests/Cargo.toml b/tests/Cargo.toml index f7eee270..cce3410f 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -22,7 +22,7 @@ managesieve = { path = "../crates/managesieve", features = ["test_mode"] } smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } mail-send = { git = "https://github.com/stalwartlabs/mail-send", default-features = false, features = ["cram-md5", "skip-ehlo"] } mail-auth = { git = "https://github.com/stalwartlabs/mail-auth", features = ["test"] } -sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } +sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } utils = { path = "../crates/utils", features = ["test_mode"] } jmap-client = { git = "https://github.com/stalwartlabs/jmap-client", features = ["websockets", "debug", "async"] } mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "serde_support", "ludicrous_mode"] } diff --git a/tests/resources/jmap_mail_get/headers.json b/tests/resources/jmap_mail_get/headers.json index 63fb683b..cbaf8fa8 100644 --- a/tests/resources/jmap_mail_get/headers.json +++ b/tests/resources/jmap_mail_get/headers.json @@ -1123,6 +1123,10 @@ { "name": "John (my dear friend)", "email": "jdoe@one.test" + }, + { + "name": "the end of the group", + "email": "" } ], "header:X-AddressesGroup-Single:asAddresses:all": [ @@ -1138,6 +1142,10 @@ { "name": "John (my dear friend)", "email": "jdoe@one.test" + }, + { + "name": "the end of the group", + "email": "" } ] ], @@ -1161,7 +1169,12 @@ }, { "name": null, - "addresses": [] + "addresses": [ + { + "name": "the end of the group", + "email": "" + } + ] } ], "header:X-AddressesGroup-Single:asGroupedAddresses:all": [ @@ -1185,7 +1198,12 @@ }, { "name": null, - "addresses": [] + "addresses": [ + { + "name": "the end of the group", + "email": "" + } + ] } ] ], diff --git a/tests/resources/jmap_mail_parse/headers.json b/tests/resources/jmap_mail_parse/headers.json index eef7ad64..7af7d153 100644 --- a/tests/resources/jmap_mail_parse/headers.json +++ b/tests/resources/jmap_mail_parse/headers.json @@ -1089,6 +1089,10 @@ { "name": "John (my dear friend)", "email": "jdoe@one.test" + }, + { + "name": "the end of the group", + "email": "" } ], "header:X-AddressesGroup-Single:asAddresses:all": [ @@ -1104,6 +1108,10 @@ { "name": "John (my dear friend)", "email": "jdoe@one.test" + }, + { + "name": "the end of the group", + "email": "" } ] ], @@ -1127,7 +1135,12 @@ }, { "name": null, - "addresses": [] + "addresses": [ + { + "name": "the end of the group", + "email": "" + } + ] } ], "header:X-AddressesGroup-Single:asGroupedAddresses:all": [ @@ -1151,7 +1164,12 @@ }, { "name": null, - "addresses": [] + "addresses": [ + { + "name": "the end of the group", + "email": "" + } + ] } ] ], diff --git a/tests/resources/test_config.toml b/tests/resources/test_config.toml index 4ba01ccf..7956f576 100644 --- a/tests/resources/test_config.toml +++ b/tests/resources/test_config.toml @@ -2,36 +2,36 @@ hostname = "test.example.org" [server.listener.jmap] -bind = ["127.0.0.1:9990"] -url = "https://127.0.0.1:9990" +bind = ["0.0.0.0:9990"] +url = "https://0.0.0.0:9990" protocol = "jmap" max-connections = 8192 [server.listener.imap] -bind = ["127.0.0.1:9991"] +bind = ["0.0.0.0:143"] protocol = "imap" max-connections = 8192 [server.listener.imaptls] -bind = ["127.0.0.1:9992"] +bind = ["0.0.0.0:9992"] protocol = "imap" max-connections = 8192 tls.implicit = true [server.listener.sieve] -bind = ["127.0.0.1:9993"] +bind = ["0.0.0.0:9993"] protocol = "managesieve" max-connections = 8192 tls.implicit = true [server.listener.smtps] -bind = ['127.0.0.1:9994'] +bind = ['0.0.0.0:9994'] greeting = 'Test SMTP instance' protocol = 'smtp' tls.implicit = true [server.listener.smtp] -bind = ['127.0.0.1:9995'] +bind = ['0.0.0.0:587'] greeting = 'Test SMTP instance' protocol = 'smtp' tls.implicit = false @@ -46,7 +46,7 @@ certificate = "default" [global.tracing] method = "stdout" -level = "info" +level = "trace" [session.ehlo] reject-non-fqdn = false diff --git a/tests/src/imap/body_structure.rs b/tests/src/imap/body_structure.rs index 32bc29c6..11516521 100644 --- a/tests/src/imap/body_structure.rs +++ b/tests/src/imap/body_structure.rs @@ -28,7 +28,7 @@ use imap_proto::{ protocol::fetch::{BodyContents, DataItem, Section}, ResponseCode, StatusResponse, }; -use mail_parser::Message; +use mail_parser::MessageParser; use super::resources_dir; @@ -41,7 +41,7 @@ fn body_structure() { } let raw_message = fs::read(&file_name).unwrap(); - let message = Message::parse(&raw_message).unwrap(); + let message = MessageParser::new().parse(&raw_message).unwrap(); let mut buf = Vec::new(); // Serialize body and bodystructure diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 1616d56b..c142184b 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -32,7 +32,7 @@ use jmap::{ }; use jmap_client::client::Client; use jmap_proto::types::id::Id; -use mail_parser::{Message, MimeHeaders}; +use mail_parser::{MessageParser, MimeHeaders}; use crate::{directory::sql::create_test_user_with_email, jmap::delivery::SmtpConnection}; @@ -227,7 +227,9 @@ pub async fn import_certs_and_encrypt() { }; for algo in [Algorithm::Aes128, Algorithm::Aes256] { - let message = Message::parse(b"Subject: test\r\ntest\r\n").unwrap(); + let message = MessageParser::new() + .parse(b"Subject: test\r\ntest\r\n") + .unwrap(); assert!(!message.is_encrypted()); params.algo = algo; message.encrypt(¶ms).await.unwrap(); @@ -259,7 +261,9 @@ pub fn check_is_encrypted() { for raw_message in messages.split("---") { let is_encrypted = raw_message.contains("TRUE"); - let message = Message::parse(raw_message.trim().as_bytes()).unwrap(); + let message = MessageParser::new() + .parse(raw_message.trim().as_bytes()) + .unwrap(); assert!(message.content_type().is_some()); assert_eq!( message.is_encrypted(), diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 5e129eb4..1684fe38 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -65,6 +65,7 @@ pub async fn test(server: Arc, client: &mut Client) { // Delivering to individuals let mut lmtp = SmtpConnection::connect().await; + lmtp.ingest( "bill@example.com", &["jdoe@example.com"], diff --git a/tests/src/jmap/email_get.rs b/tests/src/jmap/email_get.rs index b5c8c29c..72073ff2 100644 --- a/tests/src/jmap/email_get.rs +++ b/tests/src/jmap/email_get.rs @@ -29,7 +29,7 @@ use jmap_client::{ email::{self, Header, HeaderForm}, }; use jmap_proto::types::id::Id; -use mail_parser::{HeaderName, RfcHeader}; +use mail_parser::HeaderName; use crate::jmap::{mailbox::destroy_all_mailboxes, replace_blob_ids}; @@ -184,10 +184,10 @@ pub fn all_headers() -> Vec { let mut properties = Vec::new(); for header in [ - HeaderName::Rfc(RfcHeader::From), - HeaderName::Rfc(RfcHeader::To), - HeaderName::Rfc(RfcHeader::Cc), - HeaderName::Rfc(RfcHeader::Bcc), + HeaderName::From, + HeaderName::To, + HeaderName::Cc, + HeaderName::Bcc, HeaderName::Other("X-Address-Single".into()), HeaderName::Other("X-Address".into()), HeaderName::Other("X-AddressList-Single".into()), @@ -228,10 +228,10 @@ pub fn all_headers() -> Vec { } for header in [ - HeaderName::Rfc(RfcHeader::ListPost), - HeaderName::Rfc(RfcHeader::ListSubscribe), - HeaderName::Rfc(RfcHeader::ListUnsubscribe), - HeaderName::Rfc(RfcHeader::ListOwner), + HeaderName::ListPost, + HeaderName::ListSubscribe, + HeaderName::ListUnsubscribe, + HeaderName::ListOwner, HeaderName::Other("X-List-Single".into()), HeaderName::Other("X-List".into()), ] { @@ -258,8 +258,8 @@ pub fn all_headers() -> Vec { } for header in [ - HeaderName::Rfc(RfcHeader::Date), - HeaderName::Rfc(RfcHeader::ResentDate), + HeaderName::Date, + HeaderName::ResentDate, HeaderName::Other("X-Date-Single".into()), HeaderName::Other("X-Date".into()), ] { @@ -286,8 +286,8 @@ pub fn all_headers() -> Vec { } for header in [ - HeaderName::Rfc(RfcHeader::MessageId), - HeaderName::Rfc(RfcHeader::References), + HeaderName::MessageId, + HeaderName::References, HeaderName::Other("X-Id-Single".into()), HeaderName::Other("X-Id".into()), ] { @@ -314,8 +314,8 @@ pub fn all_headers() -> Vec { } for header in [ - HeaderName::Rfc(RfcHeader::Subject), - HeaderName::Rfc(RfcHeader::Keywords), + HeaderName::Subject, + HeaderName::Keywords, HeaderName::Other("X-Text-Single".into()), HeaderName::Other("X-Text".into()), ] { diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 6e865895..1de43aa6 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -30,7 +30,7 @@ use jmap_client::{ email, }; use jmap_proto::types::{collection::Collection, id::Id}; -use mail_parser::RfcHeader; +use mail_parser::HeaderName; use store::{ahash::AHashMap, write::BatchBuilder}; use crate::{ @@ -192,7 +192,10 @@ pub async fn query(client: &mut Client) { ), ( Filter::and(vec![ - (email::query::Filter::header(RfcHeader::Comments.to_string(), Some("attributed"))), + (email::query::Filter::header( + HeaderName::Comments.to_string(), + Some("attributed"), + )), (email::query::Filter::from("john")), (email::query::Filter::cc("oil")), ]), diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index 20ae7bf9..8595b30a 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -24,7 +24,7 @@ use std::{fs, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; use mail_auth::AuthenticatedMessage; -use mail_parser::Message; +use mail_parser::MessageParser; use serde::Deserialize; use smtp::{ config::{ConfigContext, IfBlock, Milter}, @@ -401,7 +401,7 @@ async fn milter_client_test() { client.init().await.unwrap(); let raw_message = load_test_message("arc", "messages"); - let message = Message::parse(raw_message.as_bytes()).unwrap(); + let message = MessageParser::new().parse(raw_message.as_bytes()).unwrap(); let r = client .connection( diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs index 2aea39e7..f084ddcb 100644 --- a/tests/src/smtp/outbound/dane.rs +++ b/tests/src/smtp/outbound/dane.rs @@ -232,7 +232,7 @@ async fn dane_test() { let r = Resolvers { dns: Resolver::new_cloudflare().unwrap(), dnssec: DnssecResolver { - resolver: AsyncResolver::tokio(conf, opts).unwrap(), + resolver: AsyncResolver::tokio(conf, opts), }, cache: smtp::core::DnsCache { tlsa: LruCache::with_capacity(10),