diff --git a/Cargo.lock b/Cargo.lock index 2dc884e8..04a6bade 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1005,10 +1005,10 @@ dependencies = [ "mail-send", "md5", "nlp", - "opentelemetry 0.22.0", - "opentelemetry-otlp 0.15.0", - "opentelemetry-semantic-conventions 0.14.0", - "opentelemetry_sdk 0.22.1", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "parking_lot", "pem", "privdrop", @@ -1032,7 +1032,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-journald", - "tracing-opentelemetry 0.23.0", + "tracing-opentelemetry", "tracing-subscriber", "unicode-security", "utils", @@ -2669,6 +2669,7 @@ name = "imap" version = "0.6.0" dependencies = [ "ahash 0.8.11", + "common", "dashmap", "directory", "imap_proto", @@ -2872,6 +2873,7 @@ dependencies = [ "bincode", "cbc", "chrono", + "common", "dashmap", "directory", "ece", @@ -3272,6 +3274,7 @@ dependencies = [ name = "mail-server" version = "0.6.0" dependencies = [ + "common", "directory", "imap", "jemallocator", @@ -3291,6 +3294,7 @@ version = "0.6.0" dependencies = [ "ahash 0.8.11", "bincode", + "common", "directory", "imap", "imap_proto", @@ -3774,22 +3778,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "opentelemetry" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" -dependencies = [ - "futures-core", - "futures-sink", - "indexmap 2.2.5", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - [[package]] name = "opentelemetry" version = "0.22.0" @@ -3805,19 +3793,6 @@ dependencies = [ "urlencoding", ] -[[package]] -name = "opentelemetry-http" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f51189ce8be654f9b5f7e70e49967ed894e84a06fc35c6c042e64ac1fc5399e" -dependencies = [ - "async-trait", - "bytes", - "http 0.2.12", - "opentelemetry 0.21.0", - "reqwest 0.11.26", -] - [[package]] name = "opentelemetry-http" version = "0.11.0" @@ -3827,31 +3802,10 @@ dependencies = [ "async-trait", "bytes", "http 0.2.12", - "opentelemetry 0.22.0", + "opentelemetry", "reqwest 0.11.26", ] -[[package]] -name = "opentelemetry-otlp" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930" -dependencies = [ - "async-trait", - "futures-core", - "http 0.2.12", - "opentelemetry 0.21.0", - "opentelemetry-http 0.10.0", - "opentelemetry-proto 0.4.0", - "opentelemetry-semantic-conventions 0.13.0", - "opentelemetry_sdk 0.21.2", - "prost 0.11.9", - "reqwest 0.11.26", - "thiserror", - "tokio", - "tonic 0.9.2", -] - [[package]] name = "opentelemetry-otlp" version = "0.15.0" @@ -3861,28 +3815,16 @@ dependencies = [ "async-trait", "futures-core", "http 0.2.12", - "opentelemetry 0.22.0", - "opentelemetry-http 0.11.0", - "opentelemetry-proto 0.5.0", - "opentelemetry-semantic-conventions 0.14.0", - "opentelemetry_sdk 0.22.1", - "prost 0.12.3", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-proto", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "prost", "reqwest 0.11.26", "thiserror", "tokio", - "tonic 0.11.0", -] - -[[package]] -name = "opentelemetry-proto" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" -dependencies = [ - "opentelemetry 0.21.0", - "opentelemetry_sdk 0.21.2", - "prost 0.11.9", - "tonic 0.9.2", + "tonic", ] [[package]] @@ -3891,19 +3833,10 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a8fddc9b68f5b80dae9d6f510b88e02396f006ad48cac349411fbecc80caae4" dependencies = [ - "opentelemetry 0.22.0", - "opentelemetry_sdk 0.22.1", - "prost 0.12.3", - "tonic 0.11.0", -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" -dependencies = [ - "opentelemetry 0.21.0", + "opentelemetry", + "opentelemetry_sdk", + "prost", + "tonic", ] [[package]] @@ -3912,28 +3845,6 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9ab5bd6c42fb9349dcf28af2ba9a0667f697f9bdcca045d39f2cec5543e2910" -[[package]] -name = "opentelemetry_sdk" -version = "0.21.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f16aec8a98a457a52664d69e0091bac3a0abd18ead9b641cb00202ba4e0efe4" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "once_cell", - "opentelemetry 0.21.0", - "ordered-float", - "percent-encoding", - "rand", - "thiserror", - "tokio", - "tokio-stream", -] - [[package]] name = "opentelemetry_sdk" version = "0.22.1" @@ -3947,7 +3858,7 @@ dependencies = [ "futures-util", "glob", "once_cell", - "opentelemetry 0.22.0", + "opentelemetry", "ordered-float", "percent-encoding", "rand", @@ -4379,16 +4290,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prost" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" -dependencies = [ - "bytes", - "prost-derive 0.11.9", -] - [[package]] name = "prost" version = "0.12.3" @@ -4396,20 +4297,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" dependencies = [ "bytes", - "prost-derive 0.12.3", -] - -[[package]] -name = "prost-derive" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d2d8d10f3c6ded6da8b05b5fb3b8a5082514344d56c9f871412d29b4e075b4" -dependencies = [ - "anyhow", - "itertools 0.10.5", - "proc-macro2", - "quote", - "syn 1.0.109", + "prost-derive", ] [[package]] @@ -5498,9 +5386,9 @@ dependencies = [ [[package]] name = "serial_test" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e56dd856803e253c8f298af3f4d7eb0ae5e23a737252cd90bb4f3b435033b2d" +checksum = "953ad9342b3aaca7cb43c45c097dd008d4907070394bd0751a0aa8817e5a018d" dependencies = [ "dashmap", "futures", @@ -5512,9 +5400,9 @@ dependencies = [ [[package]] name = "serial_test_derive" -version = "2.0.0" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" +checksum = "b93fb4adc70021ac1b47f7d45e8cc4169baaa7ea58483bc5b721d19a26202212" dependencies = [ "proc-macro2", "quote", @@ -5681,6 +5569,7 @@ dependencies = [ "ahash 0.8.11", "bincode", "blake3", + "common", "dashmap", "decancer", "directory", @@ -6077,9 +5966,10 @@ version = "0.1.0" dependencies = [ "ahash 0.8.11", "async-trait", - "base64 0.21.7", + "base64 0.22.0", "bytes", "chrono", + "common", "csv", "dashmap", "directory", @@ -6360,34 +6250,6 @@ dependencies = [ "winnow", ] -[[package]] -name = "tonic" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" -dependencies = [ - "async-trait", - "axum", - "base64 0.21.7", - "bytes", - "futures-core", - "futures-util", - "h2 0.3.24", - "http 0.2.12", - "http-body 0.4.6", - "hyper 0.14.28", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost 0.11.9", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "tonic" version = "0.11.0" @@ -6406,7 +6268,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost 0.12.3", + "prost", "tokio", "tokio-stream", "tower", @@ -6514,24 +6376,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-opentelemetry" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" -dependencies = [ - "js-sys", - "once_cell", - "opentelemetry 0.21.0", - "opentelemetry_sdk 0.21.2", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", - "web-time 0.2.4", -] - [[package]] name = "tracing-opentelemetry" version = "0.23.0" @@ -6540,14 +6384,14 @@ checksum = "a9be14ba1bbe4ab79e9229f7f89fab8d120b865859f10527f31c033e599d2284" dependencies = [ "js-sys", "once_cell", - "opentelemetry 0.22.0", - "opentelemetry_sdk 0.22.1", + "opentelemetry", + "opentelemetry_sdk", "smallvec", "tracing", "tracing-core", "tracing-log", "tracing-subscriber", - "web-time 1.1.0", + "web-time", ] [[package]] @@ -6756,7 +6600,6 @@ name = "utils" version = "0.6.0" dependencies = [ "ahash 0.8.11", - "arc-swap", "base64 0.21.7", "blake3", "chrono", @@ -6767,14 +6610,9 @@ dependencies = [ "lru-cache", "mail-auth", "mail-send", - "opentelemetry 0.21.0", - "opentelemetry-otlp 0.14.0", - "opentelemetry-semantic-conventions 0.13.0", - "opentelemetry_sdk 0.21.2", "parking_lot", "pem", "privdrop", - "proxy-header", "rand", "rcgen", "regex", @@ -6789,10 +6627,7 @@ dependencies = [ "tokio", "tokio-rustls 0.25.0", "tracing", - "tracing-appender", "tracing-journald", - "tracing-opentelemetry 0.22.0", - "tracing-subscriber", "webpki-roots 0.26.1", "x509-parser 0.16.0", ] @@ -6950,16 +6785,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "web-time" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa30049b1c872b72c89866d458eae9f20380ab280ffd1b1e18df2d3e2d98cfe0" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - [[package]] name = "web-time" version = "1.1.0" diff --git a/crates/common/src/addresses.rs b/crates/common/src/addresses.rs index 459ba0f9..3abce130 100644 --- a/crates/common/src/addresses.rs +++ b/crates/common/src/addresses.rs @@ -15,7 +15,6 @@ impl Core { directory: &Directory, email: &str, ) -> directory::Result> { - let todo = "update functions using this method."; let mut address = self .smtp .session @@ -142,8 +141,8 @@ impl AddressMapping { struct Address<'x>(&'x str); -impl<'x> ResolveVariable<'x> for Address<'x> { - fn resolve_variable(&self, _: u32) -> crate::expr::Variable<'x> { +impl ResolveVariable for Address<'_> { + fn resolve_variable(&self, _: u32) -> crate::expr::Variable { Variable::from(self.0) } } diff --git a/crates/common/src/config/jmap/capabilities.rs b/crates/common/src/config/jmap/capabilities.rs index d2b84d94..3b1cc958 100644 --- a/crates/common/src/config/jmap/capabilities.rs +++ b/crates/common/src/config/jmap/capabilities.rs @@ -11,12 +11,6 @@ use utils::{config::Config, map::vec_map::VecMap}; use super::settings::JmapConfig; -#[derive(Default)] -pub struct BaseCapabilities { - pub session: VecMap, - pub account: VecMap, -} - impl JmapConfig { pub fn add_capabilites(&mut self, config: &mut Config) { // Add core capabilities diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs index 57346cc0..d8d1a397 100644 --- a/crates/common/src/config/jmap/settings.rs +++ b/crates/common/src/config/jmap/settings.rs @@ -1,12 +1,11 @@ use std::{str::FromStr, time::Duration}; +use jmap_proto::request::capability::BaseCapabilities; use mail_parser::HeaderName; use nlp::language::Language; use store::rand::{distributions::Alphanumeric, thread_rng, Rng}; use utils::config::{cron::SimpleCron, utils::ParseValue, Config, Rate}; -use super::capabilities::BaseCapabilities; - pub struct JmapConfig { pub default_language: Language, pub query_max_results: usize, @@ -40,7 +39,6 @@ pub struct JmapConfig { pub rate_authenticated: Option, pub rate_authenticate_req: Option, pub rate_anonymous: Option, - pub rate_use_forwarded: bool, pub event_source_throttle: Duration, pub push_max_total: usize, @@ -66,6 +64,7 @@ pub struct JmapConfig { pub spam_header: Option<(HeaderName<'static>, String)>, pub http_headers: Vec<(hyper::header::HeaderName, hyper::header::HeaderValue)>, + pub http_use_forwarded: bool, pub encrypt: bool, pub encrypt_append: bool, @@ -148,9 +147,6 @@ impl JmapConfig { rate_authenticate_req: config .property_or_default_("authentication.rate-limit", "10/1m"), rate_anonymous: config.property_or_default_("jmap.rate-limit.anonymous", "100/1m"), - rate_use_forwarded: config - .property_("jmap.rate-limit.use-forwarded") - .unwrap_or(false), oauth_key: config .value("oauth.key") .map(|s| s.to_string()) @@ -216,6 +212,9 @@ impl JmapConfig { ) }) }), + http_use_forwarded: config + .property_("server.http.use-x-forwarded") + .unwrap_or(false), http_headers: config .values("server.http.headers") .map(|(_, v)| { diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 6a4fb36f..4e13b20c 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -1,7 +1,135 @@ +use std::sync::Arc; + +use arc_swap::ArcSwap; +use directory::{Directories, Directory}; +use store::{BlobBackend, BlobStore, FtsStore, LookupStore, Store, Stores}; +use utils::config::Config; + +use crate::{Core, Network}; + +use self::{ + imap::ImapConfig, jmap::settings::JmapConfig, scripts::Scripting, smtp::SmtpConfig, + storage::Storage, +}; + pub mod imap; pub mod jmap; +pub mod network; pub mod scripts; pub mod server; pub mod smtp; pub mod storage; pub mod tracers; + +impl Core { + pub async fn parse(config: &mut Config, stores: Stores) -> Self { + let mut data = config + .value_require_("storage.data") + .map(|id| id.to_string()) + .and_then(|id| { + if let Some(store) = stores.stores.get(&id) { + store.clone().into() + } else { + config.new_parse_error("storage.data", format!("Data store {id:?} not found")); + None + } + }) + .unwrap_or_default(); + let mut blob = config + .value_require_("storage.blob") + .map(|id| id.to_string()) + .and_then(|id| { + if let Some(store) = stores.blob_stores.get(&id) { + store.clone().into() + } else { + config.new_parse_error("storage.blob", format!("Blob store {id:?} not found")); + None + } + }) + .unwrap_or_default(); + let mut lookup = config + .value_require_("storage.lookup") + .map(|id| id.to_string()) + .and_then(|id| { + if let Some(store) = stores.lookup_stores.get(&id) { + store.clone().into() + } else { + config.new_parse_error( + "storage.lookup", + format!("Lookup store {id:?} not found"), + ); + None + } + }) + .unwrap_or_default(); + let mut fts = config + .value_require_("storage.fts") + .map(|id| id.to_string()) + .and_then(|id| { + if let Some(store) = stores.fts_stores.get(&id) { + store.clone().into() + } else { + config.new_parse_error( + "storage.fts", + format!("Full-text store {id:?} not found"), + ); + None + } + }) + .unwrap_or_default(); + let directories = Directories::parse(config, &stores, data.clone()).await; + let directory = config + .value_require_("storage.directory") + .map(|id| id.to_string()) + .and_then(|id| { + if let Some(directory) = directories.directories.get(&id) { + directory.clone().into() + } else { + config.new_parse_error( + "storage.directory", + format!("Directory {id:?} not found"), + ); + None + } + }) + .unwrap_or_else(|| Arc::new(Directory::default())); + + // If any of the stores are missing, disable all stores to avoid data loss + if matches!(data, Store::None) + || matches!(&blob.backend, BlobBackend::Store(Store::None)) + || matches!(lookup, LookupStore::Store(Store::None)) + || matches!(fts, FtsStore::Store(Store::None)) + { + data = Store::default(); + blob = BlobStore::default(); + lookup = LookupStore::default(); + fts = FtsStore::default(); + config.new_build_error( + "storage.*", + "One or more stores are missing, disabling all stores", + ) + } + + Self { + sieve: Scripting::parse(config, &stores).await, + network: Network::parse(config), + smtp: SmtpConfig::parse(config).await, + jmap: JmapConfig::parse(config), + imap: ImapConfig::parse(config), + storage: Storage { + data, + blob, + fts, + lookup, + lookups: stores.lookup_stores, + directory, + directories: directories.directories, + purge_schedules: stores.purge_schedules, + }, + } + } + + pub fn into_shared(self) -> Arc> { + Arc::new(ArcSwap::from_pointee(self)) + } +} diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs new file mode 100644 index 00000000..f26b4e6d --- /dev/null +++ b/crates/common/src/config/network.rs @@ -0,0 +1,45 @@ +use utils::config::Config; + +use crate::{ + expr::{if_block::IfBlock, tokenizer::TokenMap}, + listener::blocked::BlockedIps, + Network, +}; + +use super::smtp::*; + +impl Default for Network { + fn default() -> Self { + Self { + blocked_ips: Default::default(), + hostname: IfBlock::new("localhost".to_string()), + url: IfBlock::new("http://localhost:8080".to_string()), + } + } +} + +impl Network { + pub fn parse(config: &mut Config) -> Self { + let mut network = Network { + blocked_ips: BlockedIps::parse(config), + ..Default::default() + }; + let token_map = &TokenMap::default().with_smtp_variables(&[ + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_HELO_DOMAIN, + ]); + + for (value, key) in [ + (&mut network.hostname, "server.hostname"), + (&mut network.url, "server.url"), + ] { + if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { + *value = if_block; + } + } + + network + } +} diff --git a/crates/common/src/config/server/listener.rs b/crates/common/src/config/server/listener.rs index c4d79489..90558662 100644 --- a/crates/common/src/config/server/listener.rs +++ b/crates/common/src/config/server/listener.rs @@ -28,6 +28,7 @@ use rustls::{ server::ResolvesServerCert, ServerConfig, SupportedCipherSuite, ALL_VERSIONS, }; + use tokio::net::TcpSocket; use tokio_rustls::TlsAcceptor; use utils::config::{ @@ -35,21 +36,19 @@ use utils::config::{ Config, }; -use crate::{ - listener::{acme::directory::ACME_TLS_ALPN_NAME, tls::CertificateResolver, TcpAcceptor}, - ConfigBuilder, -}; +use crate::listener::{acme::directory::ACME_TLS_ALPN_NAME, tls::CertificateResolver, TcpAcceptor}; use super::{ tls::{TLS12_VERSION, TLS13_VERSION}, - Listener, Server, ServerProtocol, + Listener, Server, ServerProtocol, Servers, }; -impl ConfigBuilder { - pub fn parse_servers(&mut self, config: &mut Config) { +impl Servers { + pub fn parse(config: &mut Config) -> Self { // Parse certificates and ACME managers - self.parse_certificates(config); - self.parse_acmes(config); + let mut servers = Servers::default(); + servers.parse_certificates(config); + servers.parse_acmes(config); // Parse servers let ids = config @@ -57,8 +56,9 @@ impl ConfigBuilder { .map(|s| s.to_string()) .collect::>(); for id in ids { - self.parse_server(config, id); + servers.parse_server(config, id); } + servers } fn parse_server(&mut self, config: &mut Config, id_: String) { diff --git a/crates/common/src/config/server/mod.rs b/crates/common/src/config/server/mod.rs index bf682bb4..da1f9696 100644 --- a/crates/common/src/config/server/mod.rs +++ b/crates/common/src/config/server/mod.rs @@ -1,13 +1,22 @@ -use std::{fmt::Display, net::SocketAddr, time::Duration}; +use std::{fmt::Display, net::SocketAddr, sync::Arc, time::Duration}; +use ahash::AHashMap; use tokio::net::TcpSocket; use utils::config::ipmask::IpAddrMask; -use crate::listener::TcpAcceptor; +use crate::listener::{acme::AcmeManager, tls::Certificate, TcpAcceptor}; pub mod listener; pub mod tls; +#[derive(Default)] +pub struct Servers { + pub servers: Vec, + pub certificates: AHashMap>, + pub certificates_sni: AHashMap>, + pub acme_managers: AHashMap>, +} + #[derive(Debug, Default)] pub struct Server { pub id: String, diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index c43906dd..0543c989 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -37,18 +37,17 @@ use rustls_pemfile::{certs, read_one, Item}; use rustls_pki_types::{DnsName, PrivateKeyDer, ServerName}; use utils::config::Config; -use crate::{ - listener::{ - acme::{directory::LETS_ENCRYPT_PRODUCTION_DIRECTORY, AcmeManager}, - tls::Certificate, - }, - ConfigBuilder, +use crate::listener::{ + acme::{directory::LETS_ENCRYPT_PRODUCTION_DIRECTORY, AcmeManager}, + tls::Certificate, }; +use super::Servers; + pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; -impl ConfigBuilder { +impl Servers { pub fn parse_certificates(&mut self, config: &mut Config) { let cert_ids = config .sub_keys("certificate", ".cert") @@ -162,7 +161,6 @@ impl ConfigBuilder { domains, contact, renew_before, - self.core.storage.data.clone(), ) { Ok(acme_manager) => { self.acme_managers diff --git a/crates/common/src/config/smtp/auth.rs b/crates/common/src/config/smtp/auth.rs index 7128ead7..3fd8c8f9 100644 --- a/crates/common/src/config/smtp/auth.rs +++ b/crates/common/src/config/smtp/auth.rs @@ -386,6 +386,18 @@ impl From for Constant { } } +impl VerifyStrategy { + #[inline(always)] + pub fn verify(&self) -> bool { + matches!(self, VerifyStrategy::Strict | VerifyStrategy::Relaxed) + } + + #[inline(always)] + pub fn is_strict(&self) -> bool { + matches!(self, VerifyStrategy::Strict) + } +} + impl ParseValue for VerifyStrategy { fn parse_value(key: impl AsKey, value: &str) -> Result { match value { diff --git a/crates/common/src/config/smtp/mod.rs b/crates/common/src/config/smtp/mod.rs index d1c63b0d..1ccc909b 100644 --- a/crates/common/src/config/smtp/mod.rs +++ b/crates/common/src/config/smtp/mod.rs @@ -68,6 +68,18 @@ pub const VARIABLES_MAP: &[(&str, u32)] = &[ ("priority", V_PRIORITY), ]; +impl SmtpConfig { + pub async fn parse(config: &mut Config) -> Self { + Self { + session: SessionConfig::parse(config), + queue: QueueConfig::parse(config), + resolvers: Resolvers::parse(config).await, + mail_auth: MailAuthConfig::parse(config), + report: ReportConfig::parse(config), + } + } +} + impl TokenMap { pub fn with_smtp_variables(mut self, variables: &[u32]) -> Self { for (name, idx) in VARIABLES_MAP { diff --git a/crates/common/src/config/smtp/queue.rs b/crates/common/src/config/smtp/queue.rs index ddf5af47..3672ad96 100644 --- a/crates/common/src/config/smtp/queue.rs +++ b/crates/common/src/config/smtp/queue.rs @@ -5,10 +5,13 @@ use mail_auth::IpLookupStrategy; use mail_send::Credentials; use utils::config::{ utils::{AsKey, ParseValue}, - Config, ServerProtocol, + Config, }; -use crate::expr::{if_block::IfBlock, Constant, ConstantValue, Expression, Variable}; +use crate::{ + config::server::ServerProtocol, + expr::{if_block::IfBlock, Constant, ConstantValue, Expression, Variable}, +}; use self::throttle::{parse_throttle, parse_throttle_key}; @@ -293,10 +296,59 @@ impl QueueConfig { // Parse queue quotas and throttles queue.throttle = parse_queue_throttle(config); queue.quota = parse_queue_quota(config); + + // Parse relay hosts + queue.relay_hosts = config + .sub_keys("remote", ".address") + .map(|id| id.to_string()) + .collect::>() + .into_iter() + .filter_map(|id| parse_relay_host(config, &id).map(|host| (id, host))) + .collect(); + + // Add local delivery host + queue.relay_hosts.insert( + "local".to_string(), + RelayHost { + address: String::new(), + port: 0, + protocol: ServerProtocol::Http, + tls_implicit: Default::default(), + tls_allow_invalid_certs: Default::default(), + auth: None, + }, + ); + queue } } +fn parse_relay_host(config: &mut Config, id: &str) -> Option { + Some(RelayHost { + address: config.property_require_(("remote", id, "address"))?, + port: config + .property_require_(("remote", id, "port")) + .unwrap_or(25), + protocol: config + .property_require_(("remote", id, "protocol")) + .unwrap_or(ServerProtocol::Smtp), + auth: if let (Some(username), Some(secret)) = ( + config.value(("remote", id, "auth.username")), + config.value(("remote", id, "auth.secret")), + ) { + Credentials::new(username.to_string(), secret.to_string()).into() + } else { + None + }, + tls_implicit: config + .property_(("remote", id, "tls.implicit")) + .unwrap_or(true), + tls_allow_invalid_certs: config + .property_(("remote", id, "tls.allow-invalid-certs")) + .unwrap_or(false), + }) +} + fn parse_queue_throttle(config: &mut Config) -> QueueThrottle { // Parse throttle let mut throttle = QueueThrottle { @@ -541,3 +593,15 @@ impl ConstantValue for IpLookupStrategy { .add_constant("ipv4_then_ipv6", IpLookupStrategy::Ipv4thenIpv6); } } + +impl std::fmt::Debug for RelayHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RelayHost") + .field("address", &self.address) + .field("port", &self.port) + .field("protocol", &self.protocol) + .field("tls_implicit", &self.tls_implicit) + .field("tls_allow_invalid_certs", &self.tls_allow_invalid_certs) + .finish() + } +} diff --git a/crates/common/src/config/smtp/report.rs b/crates/common/src/config/smtp/report.rs index b416a8ef..24507fdd 100644 --- a/crates/common/src/config/smtp/report.rs +++ b/crates/common/src/config/smtp/report.rs @@ -1,11 +1,8 @@ use std::time::Duration; -use utils::{ - config::{ - utils::{AsKey, ParseValue}, - Config, - }, - snowflake::SnowflakeIdGenerator, +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, }; use crate::expr::{if_block::IfBlock, tokenizer::TokenMap, Constant, ConstantValue, Variable}; @@ -27,7 +24,6 @@ pub struct ReportAnalysis { pub addresses: Vec, pub forward: bool, pub store: Option, - pub report_id: SnowflakeIdGenerator, } pub enum AddressMatch { @@ -109,10 +105,6 @@ impl ReportConfig { .collect(), forward: config.property_("report.analysis.forward").unwrap_or(true), store: config.property_("report.analysis.store"), - report_id: config - .property_::("storage.cluster.node-id") - .map(SnowflakeIdGenerator::with_node_id) - .unwrap_or_default(), }, dkim: Report::parse(config, "dkim", &default_hostname, &sender_vars), spf: Report::parse(config, "spf", &default_hostname, &sender_vars), @@ -212,7 +204,6 @@ impl Default for ReportConfig { addresses: Default::default(), forward: true, store: None, - report_id: SnowflakeIdGenerator::new(), }, dkim: Default::default(), spf: Default::default(), diff --git a/crates/common/src/config/smtp/session.rs b/crates/common/src/config/smtp/session.rs index f69c8165..86575516 100644 --- a/crates/common/src/config/smtp/session.rs +++ b/crates/common/src/config/smtp/session.rs @@ -180,6 +180,8 @@ impl SessionConfig { V_LOCAL_IP, V_HELO_DOMAIN, ]); + let mt_priority_vars = has_sender_vars.clone().with_constants::(); + let mechanisms_vars = has_ehlo_hars.clone().with_constants::(); let mut session = SessionConfig::default(); session.rcpt.catch_all = AddressMapping::parse(config, "session.rcpt.catch-all"); @@ -200,6 +202,225 @@ impl SessionConfig { .collect(); session.throttle = SessionThrottle::parse(config); + for (value, key, token_map) in [ + (&mut session.duration, "session.duration", &has_conn_vars), + ( + &mut session.transfer_limit, + "session.transfer-limit", + &has_conn_vars, + ), + (&mut session.timeout, "session.timeout", &has_conn_vars), + ( + &mut session.connect.script, + "session.connect.script", + &has_conn_vars, + ), + ( + &mut session.connect.greeting, + "session.connect.greeting", + &has_conn_vars, + ), + ( + &mut session.extensions.pipelining, + "session.extensions.pipelining", + &has_sender_vars, + ), + ( + &mut session.extensions.dsn, + "session.extensions.dsn", + &has_sender_vars, + ), + ( + &mut session.extensions.vrfy, + "session.extensions.vrfy", + &has_sender_vars, + ), + ( + &mut session.extensions.expn, + "session.extensions.expn", + &has_sender_vars, + ), + ( + &mut session.extensions.chunking, + "session.extensions.chunking", + &has_sender_vars, + ), + ( + &mut session.extensions.requiretls, + "session.extensions.requiretls", + &has_sender_vars, + ), + ( + &mut session.extensions.no_soliciting, + "session.extensions.no-soliciting", + &has_sender_vars, + ), + ( + &mut session.extensions.future_release, + "session.extensions.future-release", + &has_sender_vars, + ), + ( + &mut session.extensions.deliver_by, + "session.extensions.deliver-by", + &has_sender_vars, + ), + ( + &mut session.extensions.mt_priority, + "session.extensions.mt-priority", + &mt_priority_vars, + ), + ( + &mut session.ehlo.script, + "session.ehlo.script", + &has_conn_vars, + ), + ( + &mut session.ehlo.require, + "session.ehlo.require", + &has_conn_vars, + ), + ( + &mut session.ehlo.reject_non_fqdn, + "session.ehlo.reject-non-fqdn", + &has_conn_vars, + ), + ( + &mut session.auth.directory, + "session.auth.directory", + &has_ehlo_hars, + ), + ( + &mut session.auth.mechanisms, + "session.auth.mechanisms", + &mechanisms_vars, + ), + ( + &mut session.auth.require, + "session.auth.require", + &has_ehlo_hars, + ), + ( + &mut session.auth.errors_max, + "session.auth.errors.max", + &has_ehlo_hars, + ), + ( + &mut session.auth.errors_wait, + "session.auth.errors.wait", + &has_ehlo_hars, + ), + ( + &mut session.auth.allow_plain_text, + "session.auth.allow-plain-text", + &has_ehlo_hars, + ), + ( + &mut session.auth.must_match_sender, + "session.auth.must-match-sender", + &has_ehlo_hars, + ), + ( + &mut session.mail.script, + "session.mail.script", + &has_sender_vars, + ), + ( + &mut session.mail.rewrite, + "session.mail.rewrite", + &has_sender_vars, + ), + ( + &mut session.rcpt.script, + "session.rcpt.script", + &has_rcpt_vars, + ), + ( + &mut session.rcpt.relay, + "session.rcpt.relay", + &has_rcpt_vars, + ), + ( + &mut session.rcpt.directory, + "session.rcpt.directory", + &has_rcpt_vars, + ), + ( + &mut session.rcpt.errors_max, + "session.rcpt.errors.max", + &has_sender_vars, + ), + ( + &mut session.rcpt.errors_wait, + "session.rcpt.errors.wait", + &has_sender_vars, + ), + ( + &mut session.rcpt.max_recipients, + "session.rcpt.max-recipients", + &has_sender_vars, + ), + ( + &mut session.rcpt.rewrite, + "session.rcpt.rewrite", + &has_rcpt_vars, + ), + ( + &mut session.data.script, + "session.data.script", + &has_rcpt_vars, + ), + ( + &mut session.data.max_messages, + "session.data.limits.messages", + &has_rcpt_vars, + ), + ( + &mut session.data.max_message_size, + "session.data.limits.size", + &has_rcpt_vars, + ), + ( + &mut session.data.max_received_headers, + "session.data.limits.received-headers", + &has_rcpt_vars, + ), + ( + &mut session.data.add_received, + "session.data.add-headers.received", + &has_rcpt_vars, + ), + ( + &mut session.data.add_received_spf, + "session.data.add-headers.received-spf", + &has_rcpt_vars, + ), + ( + &mut session.data.add_return_path, + "session.data.add-headers.return-path", + &has_rcpt_vars, + ), + ( + &mut session.data.add_auth_results, + "session.data.add-headers.auth-results", + &has_rcpt_vars, + ), + ( + &mut session.data.add_message_id, + "session.data.add-headers.message-id", + &has_rcpt_vars, + ), + ( + &mut session.data.add_date, + "session.data.add-headers.date", + &has_rcpt_vars, + ), + ] { + if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { + *value = if_block; + } + } + session } } @@ -506,7 +727,11 @@ impl From for Constant { impl ConstantValue for Mechanism { fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { - todo!() + token_map + .add_constant("login", Mechanism(AUTH_LOGIN)) + .add_constant("plain", Mechanism(AUTH_PLAIN)) + .add_constant("xoauth2", Mechanism(AUTH_XOAUTH2)) + .add_constant("oauthbearer", Mechanism(AUTH_OAUTHBEARER)); } } @@ -521,3 +746,39 @@ impl From for Mechanism { Mechanism(value) } } + +impl<'x> TryFrom> for MtPriority { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::Integer(value) => match value { + 2 => Ok(MtPriority::Mixer), + 3 => Ok(MtPriority::Stanag4406), + 4 => Ok(MtPriority::Nsep), + _ => Err(()), + }, + Variable::String(value) => MtPriority::parse_value("", &value).map_err(|_| ()), + _ => Err(()), + } + } +} + +impl From for Constant { + fn from(value: MtPriority) -> Self { + Constant::Integer(match value { + MtPriority::Mixer => 2, + MtPriority::Stanag4406 => 3, + MtPriority::Nsep => 4, + }) + } +} + +impl ConstantValue for MtPriority { + fn add_constants(token_map: &mut TokenMap) { + token_map + .add_constant("mixer", MtPriority::Mixer) + .add_constant("stanag4406", MtPriority::Stanag4406) + .add_constant("nsep", MtPriority::Nsep); + } +} diff --git a/crates/common/src/config/storage.rs b/crates/common/src/config/storage.rs index 91e131d4..14818fd7 100644 --- a/crates/common/src/config/storage.rs +++ b/crates/common/src/config/storage.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use ahash::AHashMap; use directory::Directory; -use store::{BlobStore, FtsStore, LookupStore, Store}; +use store::{write::purge::PurgeSchedule, BlobStore, FtsStore, LookupStore, Store}; pub struct Storage { pub data: Store, @@ -12,4 +12,5 @@ pub struct Storage { pub lookups: AHashMap, pub directory: Arc, pub directories: AHashMap>, + pub purge_schedules: Vec, } diff --git a/crates/common/src/expr/eval.rs b/crates/common/src/expr/eval.rs index 0a266c03..9a063eaa 100644 --- a/crates/common/src/expr/eval.rs +++ b/crates/common/src/expr/eval.rs @@ -32,10 +32,10 @@ use super::{ }; impl Core { - pub async fn eval_if TryFrom>, V: for<'x> ResolveVariable<'x>>( + pub async fn eval_if<'x, R: TryFrom>, V: ResolveVariable>( &self, - if_block: &IfBlock, - resolver: &V, + if_block: &'x IfBlock, + resolver: &'x V, ) -> Option { if if_block.is_empty() { return None; @@ -54,10 +54,10 @@ impl Core { } } - pub async fn eval_expr TryFrom>, V: for<'x> ResolveVariable<'x>>( + pub async fn eval_expr<'x, R: TryFrom>, V: ResolveVariable>( &self, - expr: &Expression, - resolver: &V, + expr: &'x Expression, + resolver: &'x V, expr_id: &str, ) -> Option { if expr.is_empty() { @@ -79,10 +79,12 @@ impl Core { } impl IfBlock { - pub async fn eval<'x, V>(&'x self, resolver: &V, core: &Core, property: &str) -> Variable<'x> - where - V: ResolveVariable<'x>, - { + pub async fn eval<'x, V: ResolveVariable>( + &'x self, + resolver: &'x V, + core: &Core, + property: &str, + ) -> Variable<'x> { let mut captures = Vec::new(); for if_then in &self.if_then { @@ -106,16 +108,13 @@ impl IfBlock { } impl Expression { - async fn eval<'x, 'y, V>( + async fn eval<'x, 'y, V: ResolveVariable>( &'x self, - resolver: &V, + resolver: &'x V, core: &Core, property: &str, captures: &'y mut Vec, - ) -> Variable<'x> - where - V: ResolveVariable<'x>, - { + ) -> Variable<'x> { let mut stack = Vec::new(); let mut exprs = self.items.iter(); diff --git a/crates/common/src/expr/functions/mod.rs b/crates/common/src/expr/functions/mod.rs index 612c6b9c..e3491cb5 100644 --- a/crates/common/src/expr/functions/mod.rs +++ b/crates/common/src/expr/functions/mod.rs @@ -31,8 +31,8 @@ pub mod email; pub mod misc; pub mod text; -pub trait ResolveVariable<'x> { - fn resolve_variable(&self, variable: u32) -> Variable<'x>; +pub trait ResolveVariable { + fn resolve_variable(&self, variable: u32) -> Variable<'_>; } impl<'x> Variable<'x> { diff --git a/crates/common/src/expr/mod.rs b/crates/common/src/expr/mod.rs index a135bc1e..56020b6e 100644 --- a/crates/common/src/expr/mod.rs +++ b/crates/common/src/expr/mod.rs @@ -21,10 +21,14 @@ * for more details. */ -use std::{borrow::Cow, time::Duration}; +use std::{ + borrow::Cow, + net::{IpAddr, Ipv4Addr, Ipv6Addr}, + time::Duration, +}; use regex::Regex; -use utils::config::utils::ParseValue; +use utils::config::{utils::ParseValue, Rate}; use self::tokenizer::TokenMap; @@ -318,3 +322,74 @@ impl From for Constant { Constant::Integer(value.as_millis() as i64) } } + +impl<'x> TryFrom> for Rate { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::Array(items) if items.len() == 2 => { + let requests = items[0].to_integer().ok_or(())?; + let period = items[1].to_integer().ok_or(())?; + + if requests > 0 && period > 0 { + Ok(Rate { + requests: requests as u64, + period: Duration::from_millis(period as u64), + }) + } else { + Err(()) + } + } + _ => Err(()), + } + } +} + +impl<'x> TryFrom> for Ipv4Addr { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::String(value) => value.parse().map_err(|_| ()), + _ => Err(()), + } + } +} + +impl<'x> TryFrom> for Ipv6Addr { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::String(value) => value.parse().map_err(|_| ()), + _ => Err(()), + } + } +} + +impl<'x> TryFrom> for IpAddr { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::String(value) => value.parse().map_err(|_| ()), + _ => Err(()), + } + } +} + +impl<'x, T: TryFrom>> TryFrom> for Vec +where + Result, ()>: FromIterator>>::Error>>, +{ + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + value + .into_array() + .into_iter() + .map(|v| T::try_from(v)) + .collect() + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index ff3d25f3..8e450227 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -1,11 +1,10 @@ -use std::{net::IpAddr, sync::Arc}; +use std::{borrow::Cow, net::IpAddr, sync::Arc}; -use ahash::AHashMap; +use arc_swap::ArcSwap; use config::{ imap::ImapConfig, jmap::settings::JmapConfig, scripts::Scripting, - server::Server, smtp::{ auth::{ArcSealer, DkimSigner}, queue::RelayHost, @@ -16,7 +15,7 @@ use config::{ }; use directory::{Directory, Principal, QueryBy}; use expr::if_block::IfBlock; -use listener::{acme::AcmeManager, blocked::BlockedIps, tls::Certificate}; +use listener::blocked::BlockedIps; use mail_send::Credentials; use opentelemetry::KeyValue; use opentelemetry_sdk::{ @@ -26,9 +25,11 @@ use opentelemetry_sdk::{ use opentelemetry_semantic_conventions::resource::{SERVICE_NAME, SERVICE_VERSION}; use sieve::Sieve; use store::LookupStore; +use tokio::sync::oneshot; use tracing::{level_filters::LevelFilter, Level}; use tracing_appender::non_blocking::WorkerGuard; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer}; +use utils::{config::Config, BlobHash}; pub mod addresses; pub mod config; @@ -39,6 +40,8 @@ pub mod scripts; pub static USER_AGENT: &str = concat!("StalwartMail/", env!("CARGO_PKG_VERSION"),); pub static DAEMON_NAME: &str = concat!("Stalwart Mail Server v", env!("CARGO_PKG_VERSION"),); +pub type SharedCore = Arc>; + pub struct Core { pub storage: Storage, pub sieve: Scripting, @@ -54,21 +57,41 @@ pub struct Network { pub url: IfBlock, } -pub struct ConfigBuilder { - pub servers: Vec, - pub certificates: AHashMap>, - pub certificates_sni: AHashMap>, - pub acme_managers: AHashMap>, - pub tracers: Vec, - pub core: Core, -} - pub enum AuthResult { Success(T), Failure, Banned, } +#[derive(Debug)] +pub enum DeliveryEvent { + Ingest { + message: IngestMessage, + result_tx: oneshot::Sender>, + }, + Stop, +} + +#[derive(Debug)] +pub struct IngestMessage { + pub sender_address: String, + pub recipients: Vec, + pub message_blob: BlobHash, + pub message_size: usize, +} + +#[derive(Debug, Clone)] +pub enum DeliveryResult { + Success, + TemporaryFailure { + reason: Cow<'static, str>, + }, + PermanentFailure { + code: [u8; 3], + reason: Cow<'static, str>, + }, +} + pub trait IntoString: Sized { fn into_string(self) -> String; } @@ -210,14 +233,8 @@ impl Core { } } -#[derive(Default)] -pub struct TracerResult { - pub guards: Vec, - pub errors: Vec, -} - impl Tracers { - pub fn enable(self) -> TracerResult { + pub fn enable(self, config: &mut Config) -> Option> { let mut layers = Vec::new(); let mut level = Level::TRACE; @@ -234,17 +251,15 @@ impl Tracers { } } - let mut result = TracerResult::default(); + let mut guards = Vec::new(); match EnvFilter::builder().parse(format!( - "smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level}" + "smtp={level},imap={level},jmap={level},store={level},common={level},utils={level},directory={level}" )) { Ok(layer) => { layers.push(layer.boxed()); } Err(err) => { - result - .errors - .push(format!("Failed to set env filter: {err}")); + config.new_build_error("tracer", format!("Failed to set env filter: {err}")); } } @@ -264,7 +279,7 @@ impl Tracers { ansi, } => { let (non_blocking, guard) = tracing_appender::non_blocking(appender); - result.guards.push(guard); + guards.push(guard); layers.push( tracing_subscriber::fmt::layer() .with_writer(non_blocking) @@ -305,9 +320,10 @@ impl Tracers { ); } Err(err) => { - result - .errors - .push(format!("Failed to start OpenTelemetry: {err}")); + config.new_build_error( + "tracer", + format!("Failed to start OpenTelemetry: {err}"), + ); } } } @@ -321,29 +337,35 @@ impl Tracers { ); } Err(err) => { - result - .errors - .push(format!("Failed to start Journald: {err}")); + config.new_build_error( + "tracer", + format!("Failed to start Journald: {err}"), + ); } } } #[cfg(not(unix))] { - result - .errors - .push("Journald is only available on Unix systems.".to_string()); + config.new_build_error( + "tracer", + "Journald is only available on Unix systems.", + ); } } } } - if let Err(err) = tracing_subscriber::registry().with(layers).try_init() { - result - .errors - .push(format!("Failed to start tracing: {err}")); + if layers.len() > 1 { + match tracing_subscriber::registry().with(layers).try_init() { + Ok(_) => Some(guards), + Err(err) => { + config.new_build_error("tracer", format!("Failed to start tracing: {err}")); + None + } + } + } else { + None } - - result } } diff --git a/crates/common/src/listener/acme/cache.rs b/crates/common/src/listener/acme/cache.rs index 8d3d0cff..893ffb82 100644 --- a/crates/common/src/listener/acme/cache.rs +++ b/crates/common/src/listener/acme/cache.rs @@ -59,7 +59,12 @@ impl AcmeManager { class: &str, items: &[String], ) -> Result>, std::io::Error> { - match self.store.config_get(self.build_key(class, items)).await { + match self + .store + .load() + .config_get(self.build_key(class, items)) + .await + { Ok(Some(content)) => match URL_SAFE_NO_PAD.decode(content.as_bytes()) { Ok(contents) => Ok(Some(contents)), Err(err) => Err(std::io::Error::new(ErrorKind::Other, err)), @@ -76,6 +81,7 @@ impl AcmeManager { contents: impl AsRef<[u8]>, ) -> Result<(), std::io::Error> { self.store + .load() .config_set([ConfigKey { key: self.build_key(class, items), value: URL_SAFE_NO_PAD.encode(contents.as_ref()), diff --git a/crates/common/src/listener/acme/mod.rs b/crates/common/src/listener/acme/mod.rs index 5f8cd740..9e54df3f 100644 --- a/crates/common/src/listener/acme/mod.rs +++ b/crates/common/src/listener/acme/mod.rs @@ -56,7 +56,7 @@ pub struct AcmeManager { pub(crate) domains: Vec, contact: Vec, renew_before: chrono::Duration, - store: Store, + store: ArcSwap, account_key: ArcSwap>, auth_keys: Mutex>>, order_in_progress: AtomicBool, @@ -81,7 +81,6 @@ impl AcmeManager { domains: Vec, contact: Vec, renew_before: Duration, - store: Store, ) -> utils::config::Result { Ok(AcmeManager { id, @@ -97,7 +96,7 @@ impl AcmeManager { }) .collect(), renew_before: chrono::Duration::from_std(renew_before).unwrap(), - store, + store: ArcSwap::from_pointee(Store::None), account_key: ArcSwap::from_pointee(Vec::new()), auth_keys: Mutex::new(AHashMap::new()), order_in_progress: false.into(), @@ -106,7 +105,10 @@ impl AcmeManager { }) } - pub async fn init(&self) -> Result { + pub async fn init(&self, store: Store) -> Result { + // Update data store + self.store.store(Arc::new(store)); + // Load account key from cache or generate a new one if let Some(account_key) = self.load_account().await? { self.account_key.store(Arc::new(account_key)); @@ -130,14 +132,14 @@ impl AcmeManager { } pub trait SpawnAcme { - fn spawn(self, shutdown_rx: watch::Receiver); + fn spawn(self, store: Store, shutdown_rx: watch::Receiver); } impl SpawnAcme for Arc { - fn spawn(self, mut shutdown_rx: watch::Receiver) { + fn spawn(self, store: Store, mut shutdown_rx: watch::Receiver) { tokio::spawn(async move { let acme = self; - let mut renew_at = match acme.init().await { + let mut renew_at = match acme.init(store).await { Ok(renew_at) => renew_at, Err(err) => { tracing::error!( diff --git a/crates/common/src/listener/listen.rs b/crates/common/src/listener/listen.rs index fb4da7c6..70c04c1a 100644 --- a/crates/common/src/listener/listen.rs +++ b/crates/common/src/listener/listen.rs @@ -30,17 +30,18 @@ use std::{ use arc_swap::ArcSwap; use proxy_header::io::ProxiedStream; use rustls::crypto::ring::cipher_suite::TLS13_AES_128_GCM_SHA256; +use store::Store; use tokio::{ net::{TcpListener, TcpStream}, sync::watch, }; use tokio_rustls::server::TlsStream; use tracing::Span; -use utils::{config::Config, failed, UnwrapFailure}; +use utils::{config::Config, UnwrapFailure}; use crate::{ - config::server::{Listener, Server}, - ConfigBuilder, Core, + config::server::{Listener, Server, Servers}, + Core, }; use super::{ @@ -87,7 +88,19 @@ impl Server { }; // Bind socket - let listener = listener.listen(); + let listener = match listener.listen() { + Ok(listener) => listener, + Err(err) => { + tracing::error!( + event = "error", + instance = instance.id, + protocol = ?instance.protocol, + reason = %err, + "Failed to bind listener" + ); + continue; + } + }; // Spawn listener let mut shutdown_rx = instance.shutdown_rx.clone(); @@ -278,15 +291,17 @@ impl SocketOpts { } } -impl ConfigBuilder { - pub fn bind(&self, config: &Config) { +impl Servers { + pub fn bind_and_drop_priv(&self, config: &mut Config) { // Bind as root for server in &self.servers { for listener in &server.listeners { - listener - .socket - .bind(listener.addr) - .failed(&format!("Failed to bind to {}", listener.addr)); + if let Err(err) = listener.socket.bind(listener.addr) { + config.new_build_error( + format!("server.listener.{}", server.id), + format!("Failed to bind to {}: {}", listener.addr, err), + ); + } } } @@ -306,7 +321,8 @@ impl ConfigBuilder { pub fn spawn( self, spawn: impl Fn(Server, watch::Receiver), - ) -> (watch::Sender, watch::Receiver) { + store: Store, + ) -> watch::Sender { // Spawn listeners let (shutdown_tx, shutdown_rx) = watch::channel(false); for server in self.servers { @@ -315,18 +331,18 @@ impl ConfigBuilder { // Spawn ACME managers for (_, acme_manager) in self.acme_managers { - acme_manager.spawn(shutdown_rx.clone()); + acme_manager.spawn(store.clone(), shutdown_rx.clone()); } - (shutdown_tx, shutdown_rx) + shutdown_tx } } impl Listener { - pub fn listen(self) -> TcpListener { + pub fn listen(self) -> Result { self.socket .listen(self.backlog.unwrap_or(1024)) - .unwrap_or_else(|err| failed(&format!("Failed to listen on {}: {}", self.addr, err))) + .map_err(|err| format!("Failed to listen on {}: {}", self.addr, err)) } } diff --git a/crates/common/src/listener/mod.rs b/crates/common/src/listener/mod.rs index bce712a0..ff4cf636 100644 --- a/crates/common/src/listener/mod.rs +++ b/crates/common/src/listener/mod.rs @@ -32,7 +32,13 @@ use tokio::{ use tokio_rustls::{Accept, TlsAcceptor}; use utils::config::ipmask::IpAddrMask; -use crate::config::server::ServerProtocol; +use crate::{ + config::{ + server::ServerProtocol, + smtp::{V_LISTENER, V_LOCAL_IP, V_REMOTE_IP}, + }, + expr::functions::ResolveVariable, +}; use self::{ acme::AcmeManager, @@ -144,6 +150,26 @@ pub trait SessionManager: Sync + Send + 'static + Clone { fn shutdown(&self) -> impl std::future::Future + Send; } +impl ResolveVariable for ServerInstance { + fn resolve_variable(&self, variable: u32) -> crate::expr::Variable<'_> { + match variable { + V_LISTENER => self.id.as_str().into(), + _ => crate::expr::Variable::default(), + } + } +} + +impl ResolveVariable for SessionData { + fn resolve_variable(&self, variable: u32) -> crate::expr::Variable<'_> { + match variable { + V_REMOTE_IP => self.remote_ip.to_string().into(), + V_LOCAL_IP => self.local_ip.to_string().into(), + V_LISTENER => self.instance.id.as_str().into(), + _ => crate::expr::Variable::default(), + } + } +} + impl Debug for TcpAcceptor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 30de0822..26dc2261 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -136,6 +136,15 @@ impl Principal { } } +impl Default for Directory { + fn default() -> Self { + Self { + store: DirectoryInner::Internal(Store::None), + cache: None, + } + } +} + impl Debug for Directory { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Directory").finish() diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index bd5380ca..89cbcb18 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -10,6 +10,7 @@ jmap = { path = "../jmap" } jmap_proto = { path = "../jmap-proto" } directory = { path = "../directory" } store = { path = "../store" } +common = { path = "../common" } nlp = { path = "../nlp" } utils = { path = "../utils" } mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } diff --git a/crates/imap/src/core/client.rs b/crates/imap/src/core/client.rs index a8d02ab8..ccf9ca61 100644 --- a/crates/imap/src/core/client.rs +++ b/crates/imap/src/core/client.rs @@ -23,14 +23,14 @@ use std::{iter::Peekable, sync::Arc, vec::IntoIter}; +use common::listener::{limiter::ConcurrencyLimiter, SessionStream}; use imap_proto::{ receiver::{self, Request}, Command, ResponseCode, StatusResponse, }; use jmap::auth::rate_limit::ConcurrencyLimiters; -use utils::listener::{limiter::ConcurrencyLimiter, SessionStream}; -use super::{SelectedMailbox, Session, SessionData, State, IMAP}; +use super::{SelectedMailbox, Session, SessionData, State}; impl Session { pub async fn ingest(&mut self, bytes: &[u8]) -> crate::Result { @@ -227,26 +227,26 @@ impl Session { let state = &self.state; // Rate limit request if let State::Authenticated { data } | State::Selected { data, .. } = state { - match data - .jmap - .lookup_store - .is_rate_allowed( - format!("ireq:{}", data.account_id).as_bytes(), - &self.imap.rate_requests, - true, - ) - .await - { - Ok(None) => {} - Ok(Some(_)) => { - return Err(StatusResponse::no("Too many requests") - .with_tag(request.tag) - .with_code(ResponseCode::Limit)); - } - Err(_) => { - return Err(StatusResponse::no("Internal server error") - .with_tag(request.tag) - .with_code(ResponseCode::ContactAdmin)); + if let Some(rate) = &self.jmap.core.imap.rate_requests { + match data + .jmap + .core + .storage + .lookup + .is_rate_allowed(format!("ireq:{}", data.account_id).as_bytes(), rate, true) + .await + { + Ok(None) => {} + Ok(Some(_)) => { + return Err(StatusResponse::no("Too many requests") + .with_tag(request.tag) + .with_code(ResponseCode::Limit)); + } + Err(_) => { + return Err(StatusResponse::no("Internal server error") + .with_tag(request.tag) + .with_code(ResponseCode::ContactAdmin)); + } } } } @@ -273,7 +273,7 @@ impl Session { } Command::Login => { if let State::NotAuthenticated { .. } = state { - if self.is_tls || self.imap.allow_plain_auth { + if self.is_tls || self.jmap.core.imap.allow_plain_auth { Ok(request) } else { Err( @@ -344,6 +344,23 @@ impl Session { }, } } + + pub fn get_concurrency_limiter(&self, account_id: u32) -> Option> { + let rate = self.jmap.core.imap.rate_concurrent?; + self.imap + .rate_limiter + .get(&account_id) + .map(|limiter| limiter.clone()) + .unwrap_or_else(|| { + let limiter = Arc::new(ConcurrencyLimiters { + concurrent_requests: ConcurrencyLimiter::new(rate), + concurrent_uploads: ConcurrencyLimiter::new(rate), + }); + self.imap.rate_limiter.insert(account_id, limiter.clone()); + limiter + }) + .into() + } } impl State { @@ -392,19 +409,3 @@ impl State { matches!(self, State::Selected { .. }) } } - -impl IMAP { - pub fn get_concurrency_limiter(&self, account_id: u32) -> Arc { - self.rate_limiter - .get(&account_id) - .map(|limiter| limiter.clone()) - .unwrap_or_else(|| { - let limiter = Arc::new(ConcurrencyLimiters { - concurrent_requests: ConcurrencyLimiter::new(self.rate_concurrent), - concurrent_uploads: ConcurrencyLimiter::new(self.rate_concurrent), - }); - self.rate_limiter.insert(account_id, limiter.clone()); - limiter - }) - } -} diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index c6f846a6..6b4cc3a4 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -4,6 +4,7 @@ use std::{ }; use ahash::AHashMap; +use common::listener::{limiter::InFlight, SessionStream}; use directory::QueryBy; use imap_proto::{protocol::list::Attribute, StatusResponse}; use jmap::{ @@ -16,10 +17,7 @@ use jmap_proto::{ }; use parking_lot::Mutex; use store::query::log::{Change, Query}; -use utils::{ - listener::{limiter::InFlight, SessionStream}, - lru_cache::LruCached, -}; +use utils::lru_cache::LruCached; use super::{Account, AccountId, Mailbox, MailboxId, MailboxSync, Session, SessionData}; @@ -27,7 +25,7 @@ impl SessionData { pub async fn new( session: &Session, access_token: &AccessToken, - in_flight: InFlight, + in_flight: Option, ) -> crate::Result { let mut session = SessionData { stream_tx: session.stream_tx.clone(), @@ -53,9 +51,11 @@ impl SessionData { account_id, format!( "{}/{}", - session.imap.name_shared, + session.jmap.core.imap.name_shared, session .jmap + .core + .storage .directory .query(QueryBy::Id(account_id), false) .await @@ -90,13 +90,17 @@ impl SessionData { ) -> crate::Result { let state_mailbox = self .jmap - .store + .core + .storage + .data .get_last_change_id(account_id, Collection::Mailbox) .await .map_err(|_| {})?; let state_email = self .jmap - .store + .core + .storage + .data .get_last_change_id(account_id, Collection::Email) .await .map_err(|_| {})?; @@ -347,8 +351,10 @@ impl SessionData { for account_id in added_account_ids { let prefix = format!( "{}/{}", - self.imap.name_shared, + self.jmap.core.imap.name_shared, self.jmap + .core + .storage .directory .query(QueryBy::Id(account_id), false) .await @@ -407,7 +413,7 @@ impl SessionData { // Only child changes, no need to re-fetch mailboxes let state_email = self .jmap - .store + .core.storage.data .get_last_change_id(account_id, Collection::Email) .await.map_err( |e| { @@ -459,8 +465,10 @@ impl SessionData { let mailbox_prefix = if !access_token.is_primary_id(account_id) { format!( "{}/{}", - self.imap.name_shared, + self.jmap.core.imap.name_shared, self.jmap + .core + .storage .directory .query(QueryBy::Id(account_id), false) .await diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 40ec2b11..2687884d 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -24,6 +24,7 @@ use std::{collections::BTreeMap, sync::Arc}; use ahash::AHashMap; +use common::listener::SessionStream; use imap_proto::{ protocol::{expunge, select::Exists, Sequence}, StatusResponse, @@ -34,7 +35,7 @@ use jmap_proto::{ types::{collection::Collection, property::Property, value::Value}, }; use store::write::assert::HashedValue; -use utils::{listener::SessionStream, lru_cache::LruCached}; +use utils::lru_cache::LruCached; use crate::core::ImapId; @@ -62,7 +63,9 @@ impl SessionData { // Obtain current state let modseq = self .jmap - .store + .core + .storage + .data .get_last_change_id(mailbox.account_id, Collection::Email) .await .map_err(|err| { @@ -231,7 +234,9 @@ impl SessionData { // Obtain current modseq if let Ok(modseq) = self .jmap - .store + .core + .storage + .data .get_last_change_id(account_id, Collection::Email) .await { diff --git a/crates/imap/src/core/mod.rs b/crates/imap/src/core/mod.rs index 7b988239..6fc5248d 100644 --- a/crates/imap/src/core/mod.rs +++ b/crates/imap/src/core/mod.rs @@ -25,10 +25,10 @@ use std::{ collections::BTreeMap, net::IpAddr, sync::{atomic::AtomicU32, Arc}, - time::Duration, }; use ahash::AHashMap; +use common::listener::{limiter::InFlight, ServerInstance, SessionStream}; use dashmap::DashMap; use imap_proto::{ protocol::{list::Attribute, ProtocolVersion}, @@ -37,17 +37,13 @@ use imap_proto::{ }; use jmap::{ auth::{rate_limit::ConcurrencyLimiters, AccessToken}, - JMAP, + JmapInstance, JMAP, }; use tokio::{ io::{ReadHalf, WriteHalf}, sync::watch, }; -use utils::{ - config::Rate, - listener::{limiter::InFlight, ServerInstance, SessionStream}, - lru_cache::LruCache, -}; +use utils::lru_cache::LruCache; pub mod client; pub mod mailbox; @@ -56,40 +52,36 @@ pub mod session; #[derive(Clone)] pub struct ImapSessionManager { - pub jmap: Arc, - pub imap: Arc, + pub imap: ImapInstance, } impl ImapSessionManager { - pub fn new(jmap: Arc, imap: Arc) -> Self { - Self { jmap, imap } + pub fn new(imap: ImapInstance) -> Self { + Self { imap } } } -pub struct IMAP { - pub max_request_size: usize, - pub max_auth_failures: u32, - pub name_shared: String, - pub allow_plain_auth: bool, - - pub timeout_auth: Duration, - pub timeout_unauth: Duration, - pub timeout_idle: Duration, +#[derive(Clone)] +pub struct ImapInstance { + pub jmap_instance: JmapInstance, + pub imap_inner: Arc, +} +pub struct Inner { pub greeting_plain: Vec, pub greeting_tls: Vec, pub rate_limiter: DashMap>, - pub rate_requests: Rate, - pub rate_concurrent: u64, pub cache_account: LruCache>, pub cache_mailbox: LruCache>, } +pub struct IMAP {} + pub struct Session { - pub jmap: Arc, - pub imap: Arc, + pub jmap: JMAP, + pub imap: Arc, pub instance: Arc, pub receiver: Receiver, pub version: ProtocolVersion, @@ -106,13 +98,13 @@ pub struct Session { pub struct SessionData { pub account_id: u32, - pub jmap: Arc, - pub imap: Arc, + pub jmap: JMAP, + pub imap: Arc, pub span: tracing::Span, pub mailboxes: parking_lot::Mutex>, pub stream_tx: Arc>>, pub state: AtomicU32, - pub in_flight: InFlight, + pub in_flight: Option, } #[derive(Debug, Default, Clone)] diff --git a/crates/imap/src/core/session.rs b/crates/imap/src/core/session.rs index 5ca2cc2c..91bcac12 100644 --- a/crates/imap/src/core/session.rs +++ b/crates/imap/src/core/session.rs @@ -23,18 +23,19 @@ use std::{borrow::Cow, sync::Arc}; +use common::listener::{stream::NullIo, SessionData, SessionManager, SessionStream}; use imap_proto::{protocol::ProtocolVersion, receiver::Receiver}; +use jmap::JMAP; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio_rustls::server::TlsStream; -use utils::listener::{stream::NullIo, SessionManager, SessionStream}; use super::{ImapSessionManager, Session, State}; impl SessionManager for ImapSessionManager { #[allow(clippy::manual_async_fn)] - fn handle( + fn handle( self, - session: utils::listener::SessionData, + session: SessionData, ) -> impl std::future::Future + Send { async move { if let Ok(mut session) = Session::new(session, self).await { @@ -51,10 +52,6 @@ impl SessionManager for ImapSessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } - - fn is_ip_blocked(&self, addr: &std::net::IpAddr) -> bool { - self.jmap.directory.blocked_ips.is_blocked(addr) - } } impl Session { @@ -66,9 +63,9 @@ impl Session { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { - self.imap.timeout_auth + self.jmap.core.imap.timeout_auth } else { - self.imap.timeout_unauth + self.jmap.core.imap.timeout_unauth }, self.stream_rx.read(&mut buf)) => { match result { @@ -112,14 +109,14 @@ impl Session { } pub async fn new( - mut session: utils::listener::SessionData, + mut session: SessionData, manager: ImapSessionManager, ) -> Result, ()> { // Write greeting let (is_tls, greeting) = if session.stream.is_tls() { - (true, &manager.imap.greeting_tls) + (true, &manager.imap.imap_inner.greeting_tls) } else { - (false, &manager.imap.greeting_plain) + (false, &manager.imap.imap_inner.greeting_plain) }; if let Err(err) = session.stream.write_all(greeting).await { tracing::debug!(parent: &session.span, event = "error", reason = %err, "Failed to write greeting."); @@ -129,16 +126,17 @@ impl Session { // Split stream into read and write halves let (stream_rx, stream_tx) = tokio::io::split(session.stream); + let jmap = JMAP::from(manager.imap.jmap_instance); Ok(Session { - receiver: Receiver::with_max_request_size(manager.imap.max_request_size), + receiver: Receiver::with_max_request_size(jmap.core.imap.max_request_size), version: ProtocolVersion::Rev1, state: State::NotAuthenticated { auth_failures: 0 }, is_tls, is_condstore: false, is_qresync: false, - imap: manager.imap, - jmap: manager.jmap, + jmap, + imap: manager.imap.imap_inner, instance: session.instance, span: session.span, in_flight: session.in_flight, diff --git a/crates/imap/src/core/writer.rs b/crates/imap/src/core/writer.rs index 07e85f7b..eeb7f308 100644 --- a/crates/imap/src/core/writer.rs +++ b/crates/imap/src/core/writer.rs @@ -30,7 +30,6 @@ use tokio::{ }; use tokio_rustls::server::TlsStream; use tracing::debug; -use utils::listener::SessionStream; use super::{Session, SessionData}; diff --git a/crates/imap/src/lib.rs b/crates/imap/src/lib.rs index 4110f3a3..3140a845 100644 --- a/crates/imap/src/lib.rs +++ b/crates/imap/src/lib.rs @@ -21,12 +21,12 @@ * for more details. */ +use core::{ImapInstance, Inner, IMAP}; use std::{collections::hash_map::RandomState, sync::Arc}; -use crate::core::IMAP; - use dashmap::DashMap; use imap_proto::{protocol::capability::Capability, ResponseCode, StatusResponse}; +use jmap::JmapInstance; use utils::{ config::Config, lru_cache::{LruCache, LruCached}, @@ -35,30 +35,17 @@ use utils::{ pub mod core; pub mod op; -static SERVER_GREETING: &str = concat!( - "Stalwart IMAP4rev2 v", - env!("CARGO_PKG_VERSION"), - " at your service." -); +static SERVER_GREETING: &str = "Stalwart IMAP4rev2 at your service."; impl IMAP { - pub async fn init(config: &Config) -> utils::config::Result> { + pub async fn init(config: &mut Config, jmap_instance: JmapInstance) -> ImapInstance { let shard_amount = config - .property::("cache.shard")? + .property_::("cache.shard") .unwrap_or(32) .next_power_of_two() as usize; - let capacity = config.property("cache.capacity")?.unwrap_or(100); + let capacity = config.property_("cache.capacity").unwrap_or(100); - Ok(Arc::new(IMAP { - max_request_size: config.property_or_default("imap.request.max-size", "52428800")?, - max_auth_failures: config.property_or_default("imap.auth.max-failures", "3")?, - name_shared: config - .value("imap.folders.name.shared") - .unwrap_or("Shared Folders") - .to_string(), - timeout_auth: config.property_or_default("imap.timeout.authenticated", "30m")?, - timeout_unauth: config.property_or_default("imap.timeout.anonymous", "1m")?, - timeout_idle: config.property_or_default("imap.timeout.idle", "30m")?, + let inner = Inner { greeting_plain: StatusResponse::ok(SERVER_GREETING) .with_code(ResponseCode::Capability { capabilities: Capability::all_capabilities(false, false), @@ -74,16 +61,18 @@ impl IMAP { RandomState::default(), shard_amount, ), - rate_requests: config.property_or_default("imap.rate-limit.requests", "2000/1m")?, - rate_concurrent: config.property("imap.rate-limit.concurrent")?.unwrap_or(4), - allow_plain_auth: config.property_or_default("imap.auth.allow-plain-text", "false")?, cache_account: LruCache::with_capacity( - config.property("cache.account.size")?.unwrap_or(2048), + config.property_("cache.account.size").unwrap_or(2048), ), cache_mailbox: LruCache::with_capacity( - config.property("cache.mailbox.size")?.unwrap_or(2048), + config.property_("cache.mailbox.size").unwrap_or(2048), ), - })) + }; + + ImapInstance { + jmap_instance, + imap_inner: Arc::new(inner), + } } } diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index db6d101e..13ea71fc 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use common::listener::SessionStream; use directory::QueryBy; use imap_proto::{ protocol::acl::{ @@ -49,7 +50,7 @@ use jmap_proto::{ }, }; use store::write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder}; -use utils::{listener::SessionStream, map::bitmap::Bitmap}; +use utils::map::bitmap::Bitmap; use crate::core::{MailboxId, Session, SessionData}; @@ -73,6 +74,8 @@ impl Session { for item in acls { if let Some(account_name) = data .jmap + .core + .storage .directory .query(QueryBy::Id(item.account_id), false) .await @@ -244,6 +247,8 @@ impl Session { // Obtain principal id let acl_account_id = match data .jmap + .core + .storage .directory .query(QueryBy::Name(arguments.identifier.as_ref().unwrap()), false) .await @@ -399,7 +404,7 @@ impl Session { } // Invalidate ACLs - data.jmap.access_tokens.remove(&acl_account_id); + data.jmap.inner.access_tokens.remove(&acl_account_id); data.write_bytes( StatusResponse::completed(command) diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index af55b5d1..91811fd9 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -29,12 +29,11 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; +use crate::core::{ImapUidToId, MailboxId, SelectedMailbox, Session, SessionData}; +use common::listener::SessionStream; use jmap::email::ingest::IngestEmail; use jmap_proto::types::{acl::Acl, keyword::Keyword, state::StateChange, type_state::DataType}; use mail_parser::MessageParser; -use utils::listener::SessionStream; - -use crate::core::{ImapUidToId, MailboxId, SelectedMailbox, Session, SessionData}; use super::ToModSeq; @@ -133,7 +132,7 @@ impl SessionData { keywords: message.flags.into_iter().map(Keyword::from).collect(), received_at: message.received_at.map(|d| d as u64), skip_duplicates: false, - encrypt: self.jmap.config.encrypt && self.jmap.config.encrypt_append, + encrypt: self.jmap.core.jmap.encrypt && self.jmap.core.jmap.encrypt_append, }) .await { diff --git a/crates/imap/src/op/authenticate.rs b/crates/imap/src/op/authenticate.rs index 21c60dec..ea7095f5 100644 --- a/crates/imap/src/op/authenticate.rs +++ b/crates/imap/src/op/authenticate.rs @@ -21,9 +21,7 @@ * for more details. */ -use std::sync::Arc; - -use directory::AuthResult; +use common::{listener::SessionStream, AuthResult}; use imap_proto::{ protocol::{authenticate::Mechanism, capability::Capability}, receiver::{self, Request}, @@ -31,7 +29,7 @@ use imap_proto::{ }; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; -use utils::listener::SessionStream; +use std::sync::Arc; use crate::core::{Session, SessionData, State}; @@ -154,41 +152,43 @@ impl Session { if let Some(access_token) = access_token { // Enforce concurrency limits - let in_flight = self - .imap + let in_flight = match self .get_concurrency_limiter(access_token.primary_id()) - .concurrent_requests - .is_allowed(); - if let Some(in_flight) = in_flight { - // Cache access token - let access_token = Arc::new(access_token); - self.jmap.cache_access_token(access_token.clone()); + .map(|limiter| limiter.concurrent_requests.is_allowed()) + { + Some(Some(limiter)) => Some(limiter), + None => None, + Some(None) => { + self.write_bytes( + StatusResponse::bye("Too many concurrent IMAP connections.").into_bytes(), + ) + .await?; + tracing::debug!(parent: &self.span, + event = "disconnect", + "Too many concurrent connections, disconnecting.", + ); + return Err(()); + } + }; - // Create session - self.state = State::Authenticated { - data: Arc::new(SessionData::new(self, &access_token, in_flight).await?), - }; - self.write_bytes( - StatusResponse::ok("Authentication successful") - .with_code(ResponseCode::Capability { - capabilities: Capability::all_capabilities(true, self.is_tls), - }) - .with_tag(tag) - .into_bytes(), - ) - .await?; - Ok(()) - } else { - self.write_bytes( - StatusResponse::bye("Too many concurrent IMAP connections.").into_bytes(), - ) - .await?; - tracing::debug!(parent: &self.span, - event = "disconnect", - "Too many concurrent connections, disconnecting.", - ); - Err(()) - } + // Cache access token + let access_token = Arc::new(access_token); + self.jmap.cache_access_token(access_token.clone()); + + // Create session + self.state = State::Authenticated { + data: Arc::new(SessionData::new(self, &access_token, in_flight).await?), + }; + self.write_bytes( + StatusResponse::ok("Authentication successful") + .with_code(ResponseCode::Capability { + capabilities: Capability::all_capabilities(true, self.is_tls), + }) + .with_tag(tag) + .into_bytes(), + ) + .await?; + Ok(()) } else { self.write_bytes( StatusResponse::no("Authentication failed") @@ -199,7 +199,7 @@ impl Session { .await?; let auth_failures = self.state.auth_failures(); - if auth_failures < self.imap.max_auth_failures { + if auth_failures < self.jmap.core.imap.max_auth_failures { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, }; diff --git a/crates/imap/src/op/capability.rs b/crates/imap/src/op/capability.rs index 8a1813c1..034d4011 100644 --- a/crates/imap/src/op/capability.rs +++ b/crates/imap/src/op/capability.rs @@ -21,6 +21,8 @@ * for more details. */ +use crate::core::Session; +use common::listener::SessionStream; use imap_proto::{ protocol::{ capability::{Capability, Response}, @@ -30,10 +32,6 @@ use imap_proto::{ Command, StatusResponse, }; -use utils::listener::SessionStream; - -use crate::core::Session; - impl Session { pub async fn handle_capability(&mut self, request: Request) -> crate::OpResult { self.write_bytes( diff --git a/crates/imap/src/op/close.rs b/crates/imap/src/op/close.rs index 4b36553a..e62a850d 100644 --- a/crates/imap/src/op/close.rs +++ b/crates/imap/src/op/close.rs @@ -21,11 +21,9 @@ * for more details. */ -use imap_proto::{receiver::Request, Command, StatusResponse}; - -use utils::listener::SessionStream; - use crate::core::{Session, State}; +use common::listener::SessionStream; +use imap_proto::{receiver::Request, Command, StatusResponse}; impl Session { pub async fn handle_close(&mut self, request: Request) -> crate::OpResult { diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 31a6138e..47b37be8 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -28,6 +28,8 @@ use imap_proto::{ StatusResponse, }; +use crate::core::{MailboxId, SelectedMailbox, Session, SessionData}; +use common::listener::SessionStream; use jmap::{email::set::TagManager, mailbox::UidMailbox}; use jmap_proto::{ error::{method::MethodError, set::SetErrorType}, @@ -37,9 +39,6 @@ use jmap_proto::{ }, }; use store::write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, F_VALUE}; -use utils::listener::SessionStream; - -use crate::core::{MailboxId, SelectedMailbox, Session, SessionData}; impl Session { pub async fn handle_copy_move( diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index 6b8ad4fd..651a262c 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -21,6 +21,8 @@ * for more details. */ +use crate::core::{Account, Mailbox, Session, SessionData}; +use common::listener::SessionStream; use imap_proto::{ protocol::{create::Arguments, list::Attribute}, receiver::Request, @@ -35,9 +37,6 @@ use jmap_proto::{ }, }; use store::{query::Filter, write::BatchBuilder}; -use utils::listener::SessionStream; - -use crate::core::{Account, Mailbox, Session, SessionData}; impl Session { pub async fn handle_create(&mut self, requests: Vec>) -> crate::OpResult { @@ -252,13 +251,13 @@ impl SessionData { let path_item = path_item.trim(); if path_item.is_empty() { return Err(StatusResponse::no("Invalid empty path item.")); - } else if path_item.len() > self.jmap.config.mailbox_name_max_len { + } else if path_item.len() > self.jmap.core.jmap.mailbox_name_max_len { return Err(StatusResponse::no("Mailbox name is too long.")); } path.push(path_item); } - if path.len() > self.jmap.config.mailbox_max_depth { + if path.len() > self.jmap.core.jmap.mailbox_max_depth { return Err(StatusResponse::no("Mailbox path is too deep.")); } } else { @@ -272,7 +271,7 @@ impl SessionData { let (account_id, path) = { let mailboxes = self.mailboxes.lock(); let first_path_item = path.first().unwrap(); - let account = if first_path_item == &self.imap.name_shared { + let account = if first_path_item == &self.jmap.core.imap.name_shared { // Shared Folders// if path.len() < 3 { return Err(StatusResponse::no( diff --git a/crates/imap/src/op/delete.rs b/crates/imap/src/op/delete.rs index 71f24d73..01b90377 100644 --- a/crates/imap/src/op/delete.rs +++ b/crates/imap/src/op/delete.rs @@ -21,12 +21,11 @@ * for more details. */ +use crate::core::{Session, SessionData}; +use common::listener::SessionStream; use imap_proto::{protocol::delete::Arguments, receiver::Request, Command, StatusResponse}; use jmap_proto::types::{state::StateChange, type_state::DataType}; use store::write::log::ChangeLogBuilder; -use utils::listener::SessionStream; - -use crate::core::{Session, SessionData}; impl Session { pub async fn handle_delete(&mut self, requests: Vec>) -> crate::OpResult { diff --git a/crates/imap/src/op/enable.rs b/crates/imap/src/op/enable.rs index 959a7b72..43ba4e7c 100644 --- a/crates/imap/src/op/enable.rs +++ b/crates/imap/src/op/enable.rs @@ -21,16 +21,14 @@ * for more details. */ +use crate::core::Session; +use common::listener::SessionStream; use imap_proto::{ protocol::{capability::Capability, enable, ImapResponse, ProtocolVersion}, receiver::Request, Command, StatusResponse, }; -use utils::listener::SessionStream; - -use crate::core::Session; - impl Session { pub async fn handle_enable(&mut self, request: Request) -> crate::OpResult { match request.parse_enable() { diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index c7b6c910..97af3e76 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -30,6 +30,8 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; +use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; +use common::listener::SessionStream; use jmap::{email::set::TagManager, mailbox::UidMailbox}; use jmap_proto::{ error::method::MethodError, @@ -39,9 +41,6 @@ use jmap_proto::{ }, }; use store::write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, F_VALUE}; -use utils::listener::SessionStream; - -use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; use super::ToModSeq; diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 231a2433..d5c40f6c 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -23,7 +23,9 @@ use std::{borrow::Cow, sync::Arc}; +use crate::core::{SelectedMailbox, Session, SessionData}; use ahash::AHashMap; +use common::listener::SessionStream; use imap_proto::{ parser::PushUnique, protocol::{ @@ -50,9 +52,6 @@ use store::{ query::log::{Change, Query}, write::{assert::HashedValue, BatchBuilder, Bincode, F_BITMAP, F_VALUE}, }; -use utils::listener::SessionStream; - -use crate::core::{SelectedMailbox, Session, SessionData}; use super::FromModSeq; diff --git a/crates/imap/src/op/idle.rs b/crates/imap/src/op/idle.rs index 183d5e42..a0da4b71 100644 --- a/crates/imap/src/op/idle.rs +++ b/crates/imap/src/op/idle.rs @@ -35,10 +35,10 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; +use common::listener::SessionStream; use jmap_proto::types::{collection::Collection, type_state::DataType}; use store::query::log::Query; use tokio::io::AsyncReadExt; -use utils::listener::SessionStream; use utils::map::bitmap::Bitmap; use crate::core::{SelectedMailbox, Session, SessionData, State}; @@ -84,7 +84,7 @@ impl Session { let mut buf = vec![0; 1024]; loop { tokio::select! { - result = tokio::time::timeout(self.imap.timeout_idle, self.stream_rx.read(&mut buf)) => { + result = tokio::time::timeout(self.jmap.core.imap.timeout_idle, self.stream_rx.read(&mut buf)) => { match result { Ok(Ok(bytes_read)) => { if bytes_read > 0 { diff --git a/crates/imap/src/op/list.rs b/crates/imap/src/op/list.rs index 958d1380..3b946c68 100644 --- a/crates/imap/src/op/list.rs +++ b/crates/imap/src/op/list.rs @@ -21,6 +21,8 @@ * for more details. */ +use crate::core::{Session, SessionData}; +use common::listener::SessionStream; use imap_proto::{ protocol::{ list::{ @@ -32,10 +34,6 @@ use imap_proto::{ Command, StatusResponse, }; -use utils::listener::SessionStream; - -use crate::core::{Session, SessionData}; - impl Session { pub async fn handle_list(&mut self, request: Request) -> crate::OpResult { let command = request.command; @@ -179,9 +177,11 @@ impl SessionData { for account in self.mailboxes.lock().iter() { if let Some(prefix) = &account.prefix { if !added_shared_folder { - if !filter_subscribed && matches_pattern(&patterns, &self.imap.name_shared) { + if !filter_subscribed + && matches_pattern(&patterns, &self.jmap.core.imap.name_shared) + { list_items.push(ListItem { - mailbox_name: self.imap.name_shared.clone(), + mailbox_name: self.jmap.core.imap.name_shared.clone(), attributes: if include_children { vec![Attribute::HasChildren, Attribute::NoSelect] } else { diff --git a/crates/imap/src/op/login.rs b/crates/imap/src/op/login.rs index 150c68bc..6c32939a 100644 --- a/crates/imap/src/op/login.rs +++ b/crates/imap/src/op/login.rs @@ -23,10 +23,9 @@ use imap_proto::{receiver::Request, Command}; -use mail_send::Credentials; -use utils::listener::SessionStream; - use crate::core::Session; +use common::listener::SessionStream; +use mail_send::Credentials; impl Session { pub async fn handle_login(&mut self, request: Request) -> crate::OpResult { diff --git a/crates/imap/src/op/logout.rs b/crates/imap/src/op/logout.rs index e6e5b22e..9af4663c 100644 --- a/crates/imap/src/op/logout.rs +++ b/crates/imap/src/op/logout.rs @@ -21,11 +21,9 @@ * for more details. */ -use imap_proto::{receiver::Request, Command, StatusResponse}; - -use utils::listener::SessionStream; - use crate::core::Session; +use common::listener::SessionStream; +use imap_proto::{receiver::Request, Command, StatusResponse}; impl Session { pub async fn handle_logout(&mut self, request: Request) -> crate::OpResult { diff --git a/crates/imap/src/op/namespace.rs b/crates/imap/src/op/namespace.rs index 8e611956..0cd42778 100644 --- a/crates/imap/src/op/namespace.rs +++ b/crates/imap/src/op/namespace.rs @@ -21,16 +21,14 @@ * for more details. */ +use crate::core::Session; +use common::listener::SessionStream; use imap_proto::{ protocol::{namespace::Response, ImapResponse}, receiver::Request, Command, StatusResponse, }; -use utils::listener::SessionStream; - -use crate::core::Session; - impl Session { pub async fn handle_namespace(&mut self, request: Request) -> crate::OpResult { self.write_bytes( @@ -39,7 +37,7 @@ impl Session { .serialize( Response { shared_prefix: if self.state.session_data().mailboxes.lock().len() > 1 { - self.imap.name_shared.clone().into() + self.jmap.core.imap.name_shared.clone().into() } else { None }, diff --git a/crates/imap/src/op/noop.rs b/crates/imap/src/op/noop.rs index 3a989201..61cf509a 100644 --- a/crates/imap/src/op/noop.rs +++ b/crates/imap/src/op/noop.rs @@ -21,11 +21,9 @@ * for more details. */ -use imap_proto::{receiver::Request, Command, StatusResponse}; - -use utils::listener::SessionStream; - use crate::core::{Session, State}; +use common::listener::SessionStream; +use imap_proto::{receiver::Request, Command, StatusResponse}; impl Session { pub async fn handle_noop(&mut self, request: Request) -> crate::OpResult { diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index 0995cd2d..299c5f01 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -23,6 +23,8 @@ use std::collections::BTreeMap; +use crate::core::{Session, SessionData}; +use common::listener::SessionStream; use imap_proto::{ protocol::rename::Arguments, receiver::Request, Command, ResponseCode, StatusResponse, }; @@ -36,9 +38,6 @@ use jmap_proto::{ }, }; use store::write::{assert::HashedValue, BatchBuilder}; -use utils::listener::SessionStream; - -use crate::core::{Session, SessionData}; impl Session { pub async fn handle_rename(&mut self, request: Request) -> crate::OpResult { diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index 47a67582..8d539a7c 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use common::listener::SessionStream; use imap_proto::{ protocol::{ search::{self, Arguments, Filter, Response, ResultOption}, @@ -31,7 +32,6 @@ use imap_proto::{ receiver::Request, Command, StatusResponse, }; - use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::HeaderName; use nlp::language::Language; @@ -42,7 +42,6 @@ use store::{ write::now, }; use tokio::sync::watch; -use utils::listener::SessionStream; use crate::core::{ImapId, MailboxState, SavedSearch, SelectedMailbox, Session, SessionData}; @@ -153,7 +152,9 @@ impl SessionData { let is_sort = if let Some(sort) = arguments.sort { mailbox.map_search_results( self.jmap - .store + .core + .storage + .data .sort( result_set, sort.into_iter() @@ -285,7 +286,7 @@ impl SessionData { fts_filters.push(FtsFilter::has_text_detect( Field::Body, text, - self.jmap.config.default_language, + self.jmap.core.jmap.default_language, )); } search::Filter::Cc(text) => { @@ -343,7 +344,7 @@ impl SessionData { fts_filters.push(FtsFilter::has_text_detect( Field::Header(HeaderName::Subject), text, - self.jmap.config.default_language, + self.jmap.core.jmap.default_language, )); } search::Filter::Text(text) => { @@ -371,17 +372,17 @@ impl SessionData { fts_filters.push(FtsFilter::has_text_detect( Field::Header(HeaderName::Subject), &text, - self.jmap.config.default_language, + self.jmap.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( Field::Body, &text, - self.jmap.config.default_language, + self.jmap.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( Field::Attachment, text, - self.jmap.config.default_language, + self.jmap.core.jmap.default_language, )); fts_filters.push(FtsFilter::End); } diff --git a/crates/imap/src/op/select.rs b/crates/imap/src/op/select.rs index 3ac36be0..5e972ab0 100644 --- a/crates/imap/src/op/select.rs +++ b/crates/imap/src/op/select.rs @@ -34,10 +34,10 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; -use jmap_proto::types::id::Id; -use utils::{listener::SessionStream, lru_cache::LruCached}; - use crate::core::{SavedSearch, SelectedMailbox, Session, State}; +use common::listener::SessionStream; +use jmap_proto::types::id::Id; +use utils::lru_cache::LruCached; use super::ToModSeq; diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index 839ca986..9408ee73 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -23,6 +23,8 @@ use std::sync::Arc; +use crate::core::{Mailbox, Session, SessionData}; +use common::listener::SessionStream; use imap_proto::{ parser::PushUnique, protocol::status::{Status, StatusItem, StatusItemType}, @@ -39,9 +41,6 @@ use store::{ IndexKeyPrefix, IterateParams, ValueKey, }; use store::{Deserialize, U32_LEN}; -use utils::listener::SessionStream; - -use crate::core::{Mailbox, Session, SessionData}; use super::ToModSeq; @@ -95,11 +94,11 @@ impl SessionData { mailbox } else { // Some IMAP clients will try to get the status of a mailbox with the NoSelect flag - return if mailbox_name == self.imap.name_shared + return if mailbox_name == self.jmap.core.imap.name_shared || mailbox_name .split_once('/') .map_or(false, |(base_name, path)| { - base_name == self.imap.name_shared && !path.contains('/') + base_name == self.jmap.core.imap.name_shared && !path.contains('/') }) { Ok(StatusItem { @@ -238,7 +237,9 @@ impl SessionData { Status::UidNext => { (self .jmap - .store + .core + .storage + .data .get_counter(ValueKey { account_id: mailbox.account_id, collection: Collection::Mailbox.into(), @@ -388,7 +389,9 @@ impl SessionData { ) -> super::Result { let mut total_size = 0u32; self.jmap - .store + .core + .storage + .data .iterate( IterateParams::new( IndexKeyPrefix { diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 09578041..eba56e0e 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -23,7 +23,9 @@ use std::sync::Arc; +use crate::core::{message::MAX_RETRIES, SelectedMailbox, Session, SessionData}; use ahash::AHashSet; +use common::listener::SessionStream; use imap_proto::{ protocol::{ fetch::{DataItem, FetchItem}, @@ -45,9 +47,6 @@ use store::{ query::log::{Change, Query}, write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, F_VALUE}, }; -use utils::listener::SessionStream; - -use crate::core::{message::MAX_RETRIES, SelectedMailbox, Session, SessionData}; use super::FromModSeq; diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 113d2181..ccab91e2 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -21,6 +21,8 @@ * for more details. */ +use crate::core::{Session, SessionData}; +use common::listener::SessionStream; use imap_proto::{receiver::Request, Command, ResponseCode, StatusResponse}; use jmap::mailbox::set::{MailboxSubscribe, SCHEMA}; use jmap_proto::{ @@ -32,9 +34,6 @@ use jmap_proto::{ }, }; use store::write::{assert::HashedValue, BatchBuilder}; -use utils::listener::SessionStream; - -use crate::core::{Session, SessionData}; impl Session { pub async fn handle_subscribe( diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index 445b4abd..0f5d0ee6 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -23,7 +23,9 @@ use std::sync::Arc; +use crate::core::{SelectedMailbox, Session, SessionData}; use ahash::AHashMap; +use common::listener::SessionStream; use imap_proto::{ protocol::{ thread::{Arguments, Response}, @@ -33,10 +35,6 @@ use imap_proto::{ Command, StatusResponse, }; -use utils::listener::SessionStream; - -use crate::core::{SelectedMailbox, Session, SessionData}; - impl Session { pub async fn handle_thread( &mut self, diff --git a/crates/jmap-proto/src/request/capability.rs b/crates/jmap-proto/src/request/capability.rs index a767c931..d8506df7 100644 --- a/crates/jmap-proto/src/request/capability.rs +++ b/crates/jmap-proto/src/request/capability.rs @@ -26,9 +26,47 @@ use utils::map::vec_map::VecMap; use crate::{ error::request::RequestError, parser::{json::Parser, Error, JsonObjectParser}, - types::type_state::DataType, + response::serialize::serialize_hex, + types::{id::Id, type_state::DataType}, }; +#[derive(Debug, Clone, serde::Serialize)] +pub struct Session { + #[serde(rename(serialize = "capabilities"))] + capabilities: VecMap, + #[serde(rename(serialize = "accounts"))] + accounts: VecMap, + #[serde(rename(serialize = "primaryAccounts"))] + primary_accounts: VecMap, + #[serde(rename(serialize = "username"))] + username: String, + #[serde(rename(serialize = "apiUrl"))] + api_url: String, + #[serde(rename(serialize = "downloadUrl"))] + download_url: String, + #[serde(rename(serialize = "uploadUrl"))] + upload_url: String, + #[serde(rename(serialize = "eventSourceUrl"))] + event_source_url: String, + #[serde(rename(serialize = "state"))] + #[serde(serialize_with = "serialize_hex")] + state: u32, + #[serde(skip)] + base_url: String, +} + +#[derive(Debug, Clone, serde::Serialize)] +struct Account { + #[serde(rename(serialize = "name"))] + name: String, + #[serde(rename(serialize = "isPersonal"))] + is_personal: bool, + #[serde(rename(serialize = "isReadOnly"))] + is_read_only: bool, + #[serde(rename(serialize = "accountCapabilities"))] + account_capabilities: VecMap, +} + #[derive(Debug, Clone, Copy, serde::Serialize, Hash, PartialEq, Eq)] pub enum Capability { #[serde(rename(serialize = "urn:ietf:params:jmap:core"))] @@ -158,6 +196,124 @@ pub struct BlobCapabilities { #[derive(Debug, Clone, Default, serde::Serialize)] pub struct EmptyCapabilities {} +#[derive(Default)] +pub struct BaseCapabilities { + pub session: VecMap, + pub account: VecMap, +} + +impl Session { + pub fn new(base_url: impl Into, base_capabilities: &BaseCapabilities) -> Session { + let base_url = base_url.into(); + let mut capabilities = base_capabilities.session.clone(); + capabilities.append( + Capability::WebSocket, + Capabilities::WebSocket(WebSocketCapabilities::new(&base_url)), + ); + + Session { + capabilities, + accounts: VecMap::new(), + primary_accounts: VecMap::new(), + username: "".to_string(), + api_url: format!("{}/jmap/", base_url), + download_url: format!( + "{}/jmap/download/{{accountId}}/{{blobId}}/{{name}}?accept={{type}}", + base_url + ), + upload_url: format!("{}/jmap/upload/{{accountId}}/", base_url), + event_source_url: format!( + "{}/jmap/eventsource/?types={{types}}&closeafter={{closeafter}}&ping={{ping}}", + base_url + ), + base_url, + state: 0, + } + } + + pub fn set_primary_account( + &mut self, + account_id: Id, + username: String, + name: String, + capabilities: Option<&[Capability]>, + account_capabilities: &VecMap, + ) { + self.username = username; + + if let Some(capabilities) = capabilities { + for capability in capabilities { + self.primary_accounts.append(*capability, account_id); + } + } else { + for capability in self.capabilities.keys() { + self.primary_accounts.append(*capability, account_id); + } + } + + self.accounts.set( + account_id, + Account::new(name, true, false).add_capabilities(capabilities, account_capabilities), + ); + } + + pub fn add_account( + &mut self, + account_id: Id, + name: String, + is_personal: bool, + is_read_only: bool, + capabilities: Option<&[Capability]>, + account_capabilities: &VecMap, + ) { + self.accounts.set( + account_id, + Account::new(name, is_personal, is_read_only) + .add_capabilities(capabilities, account_capabilities), + ); + } + + pub fn set_state(&mut self, state: u32) { + self.state = state; + } + + pub fn api_url(&self) -> &str { + &self.api_url + } + + pub fn base_url(&self) -> &str { + &self.base_url + } +} + +impl Account { + pub fn new(name: String, is_personal: bool, is_read_only: bool) -> Account { + Account { + name, + is_personal, + is_read_only, + account_capabilities: VecMap::new(), + } + } + + pub fn add_capabilities( + mut self, + capabilities: Option<&[Capability]>, + account_capabilities: &VecMap, + ) -> Account { + if let Some(capabilities) = capabilities { + for capability in capabilities { + if let Some(value) = account_capabilities.get(capability) { + self.account_capabilities.append(*capability, value.clone()); + } + } + } else { + self.account_capabilities = account_capabilities.clone(); + } + self + } +} + impl Default for SieveSessionCapabilities { fn default() -> Self { Self { @@ -166,6 +322,15 @@ impl Default for SieveSessionCapabilities { } } +impl WebSocketCapabilities { + pub fn new(base_url: &str) -> Self { + WebSocketCapabilities { + url: format!("ws{}/jmap/ws", base_url.strip_prefix("http").unwrap()), + supports_push: true, + } + } +} + impl JsonObjectParser for Capability { fn parse(parser: &mut Parser<'_>) -> crate::parser::Result where diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 0113dfce..32494a22 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -10,6 +10,7 @@ nlp = { path = "../nlp" } jmap_proto = { path = "../jmap-proto" } smtp = { path = "../smtp" } utils = { path = "../utils" } +common = { path = "../common" } directory = { path = "../directory" } smtp-proto = { version = "0.1" } mail-parser = { version = "0.9", features = ["full_encoding", "serde_support", "ludicrous_mode"] } diff --git a/crates/jmap/src/api/admin.rs b/crates/jmap/src/api/admin.rs index 7ba33118..9ccfea02 100644 --- a/crates/jmap/src/api/admin.rs +++ b/crates/jmap/src/api/admin.rs @@ -36,7 +36,6 @@ use utils::{config::ConfigKey, url_params::UrlParams}; use crate::{ auth::{oauth::OAuthCodeRequest, AccessToken}, - services::housekeeper, JMAP, }; @@ -102,7 +101,9 @@ impl JMAP { body.and_then(|body| serde_json::from_slice::(&body).ok()) { match self - .store + .core + .storage + .data .create_account( Principal { id: principal.id, @@ -141,7 +142,7 @@ impl JMAP { let page: usize = params.parse("page").unwrap_or(0); let limit: usize = params.parse("limit").unwrap_or(0); - match self.store.list_accounts(filter, typ).await { + match self.core.storage.data.list_accounts(filter, typ).await { Ok(accounts) => { let (total, accounts) = if limit > 0 { let offset = page.saturating_sub(1) * limit; @@ -166,7 +167,7 @@ impl JMAP { } ("principal", Some(name), method) => { // Fetch, update or delete principal - let account_id = match self.store.get_account_id(name).await { + let account_id = match self.core.storage.data.get_account_id(name).await { Ok(Some(account_id)) => account_id, Ok(None) => { return RequestError::blank( @@ -183,8 +184,16 @@ impl JMAP { match *method { Method::GET => { - let result = match self.store.query(QueryBy::Id(account_id), true).await { - Ok(Some(principal)) => self.store.map_group_ids(principal).await, + let result = match self + .core + .storage + .data + .query(QueryBy::Id(account_id), true) + .await + { + Ok(Some(principal)) => { + self.core.storage.data.map_group_ids(principal).await + } Ok(None) => { return RequestError::blank( StatusCode::NOT_FOUND.as_u16(), @@ -205,11 +214,20 @@ impl JMAP { as u64; // Obtain member names - for member_id in - self.store.get_members(account_id).await.unwrap_or_default() + for member_id in self + .core + .storage + .data + .get_members(account_id) + .await + .unwrap_or_default() { - if let Ok(Some(member_principal)) = - self.store.query(QueryBy::Id(member_id), false).await + if let Ok(Some(member_principal)) = self + .core + .storage + .data + .query(QueryBy::Id(member_id), false) + .await { principal.members.push(member_principal.name); } @@ -225,7 +243,7 @@ impl JMAP { } Method::DELETE => { // Remove FTS index - if let Err(err) = self.fts_store.remove_all(account_id).await { + if let Err(err) = self.core.storage.fts.remove_all(account_id).await { tracing::warn!( context = "fts", event = "error", @@ -241,7 +259,13 @@ impl JMAP { } // Delete account - match self.store.delete_account(QueryBy::Id(account_id)).await { + match self + .core + .storage + .data + .delete_account(QueryBy::Id(account_id)) + .await + { Ok(_) => JsonResponse::new(json!({ "data": (), })) @@ -254,7 +278,9 @@ impl JMAP { serde_json::from_slice::>(&body).ok() }) { match self - .store + .core + .storage + .data .update_account(QueryBy::Id(account_id), changes) .await { @@ -283,7 +309,7 @@ impl JMAP { let page: usize = params.parse("page").unwrap_or(0); let limit: usize = params.parse("limit").unwrap_or(0); - match self.store.list_domains(filter).await { + match self.core.storage.data.list_domains(filter).await { Ok(domains) => { let (total, domains) = if limit > 0 { let offset = page.saturating_sub(1) * limit; @@ -308,7 +334,7 @@ impl JMAP { } ("domain", Some(domain), &Method::POST) => { // Create domain - match self.store.create_domain(domain).await { + match self.core.storage.data.create_domain(domain).await { Ok(_) => JsonResponse::new(json!({ "data": (), })) @@ -318,7 +344,7 @@ impl JMAP { } ("domain", Some(domain), &Method::DELETE) => { // Delete domain - match self.store.delete_domain(domain).await { + match self.core.storage.data.delete_domain(domain).await { Ok(_) => JsonResponse::new(json!({ "data": (), })) @@ -327,8 +353,14 @@ impl JMAP { } } ("store", Some("maintenance"), &Method::GET) => { - match self.store.purge_blobs(self.blob_store.clone()).await { - Ok(_) => match self.store.purge_store().await { + match self + .core + .storage + .data + .purge_blobs(self.core.storage.blob.clone()) + .await + { + Ok(_) => match self.core.storage.data.purge_store().await { Ok(_) => JsonResponse::new(json!({ "data": (), })) @@ -348,8 +380,9 @@ impl JMAP { .into_http_response(), } } - ("reload", Some("settings"), &Method::GET) => { + /*("reload", Some("settings"), &Method::GET) => { let _ = self + .inner .housekeeper_tx .send(housekeeper::Event::ReloadConfig) .await; @@ -361,6 +394,7 @@ impl JMAP { } ("reload", Some("certificates"), &Method::GET) => { let _ = self + .inner .housekeeper_tx .send(housekeeper::Event::ReloadCertificates) .await; @@ -369,7 +403,7 @@ impl JMAP { "data": (), })) .into_http_response() - } + }*/ ("settings", Some("group"), &Method::GET) => { // List settings let params = UrlParams::new(req.uri().query()); @@ -400,7 +434,7 @@ impl JMAP { params.parse::("page").unwrap_or(0).saturating_sub(1) * limit; let has_filter = !filter.is_empty(); - match self.store.config_list(&prefix, true).await { + match self.core.storage.data.config_list(&prefix, true).await { Ok(settings) => if !suffix.is_empty() && !settings.is_empty() { // Obtain record ids let mut total = 0; @@ -514,7 +548,7 @@ impl JMAP { let limit: usize = params.parse("limit").unwrap_or(0); let offset = params.parse::("page").unwrap_or(0).saturating_sub(1) * limit; - match self.store.config_list(&prefix, true).await { + match self.core.storage.data.config_list(&prefix, true).await { Ok(settings) => { let total = settings.len(); let items = settings @@ -554,7 +588,7 @@ impl JMAP { let mut results = AHashMap::with_capacity(keys.len()); for key in keys { - match self.store.config_get(key).await { + match self.core.storage.data.config_get(key).await { Ok(Some(value)) => { results.insert(key.to_string(), value); } @@ -571,7 +605,7 @@ impl JMAP { } else { prefix.to_string() }; - match self.store.config_list(&prefix, false).await { + match self.core.storage.data.config_list(&prefix, false).await { Ok(values) => { results.extend(values); } @@ -597,7 +631,7 @@ impl JMAP { } } ("settings", Some(prefix), &Method::DELETE) if !prefix.is_empty() => { - match self.store.config_clear(prefix).await { + match self.core.storage.data.config_clear(prefix).await { Ok(_) => JsonResponse::new(json!({ "data": (), })) @@ -620,15 +654,26 @@ impl JMAP { match change { UpdateSettings::Delete { keys } => { for key in keys { - result = self.store.config_clear(key).await.map(|_| true); + result = self + .core + .storage + .data + .config_clear(key) + .await + .map(|_| true); if result.is_err() { break 'next; } } } UpdateSettings::Clear { prefix } => { - result = - self.store.config_clear_prefix(&prefix).await.map(|_| true); + result = self + .core + .storage + .data + .config_clear_prefix(&prefix) + .await + .map(|_| true); if result.is_err() { break; } @@ -641,7 +686,9 @@ impl JMAP { if assert_empty { if let Some(prefix) = &prefix { result = self - .store + .core + .storage + .data .config_list(&format!("{prefix}."), true) .await .map(|items| items.is_empty()); @@ -651,7 +698,9 @@ impl JMAP { } } else if let Some((key, _)) = values.first() { result = self - .store + .core + .storage + .data .config_get(key) .await .map(|items| items.is_none()); @@ -663,7 +712,9 @@ impl JMAP { } result = self - .store + .core + .storage + .data .config_set(values.into_iter().map(|(key, value)| ConfigKey { key: if let Some(prefix) = &prefix { format!("{prefix}.{key}") diff --git a/crates/jmap/src/api/config.rs b/crates/jmap/src/api/config.rs deleted file mode 100644 index d7feeb99..00000000 --- a/crates/jmap/src/api/config.rs +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{str::FromStr, time::Duration}; - -use nlp::language::Language; -use store::rand::{distributions::Alphanumeric, thread_rng, Rng}; - -use super::session::BaseCapabilities; - -impl crate::Config { - pub fn new(settings: &utils::config::Config) -> Result { - let mut config = Self { - default_language: Language::from_iso_639( - settings - .value("storage.full-text.default-language") - .unwrap_or("en"), - ) - .unwrap_or(Language::English), - query_max_results: settings - .property("jmap.protocol.query.max-results")? - .unwrap_or(5000), - changes_max_results: settings - .property("jmap.protocol.changes.max-results")? - .unwrap_or(5000), - snippet_max_results: settings - .property("jmap.protocol.search-snippet.max-results")? - .unwrap_or(100), - request_max_size: settings - .property("jmap.protocol.request.max-size")? - .unwrap_or(10000000), - request_max_calls: settings - .property("jmap.protocol.request.max-calls")? - .unwrap_or(16), - request_max_concurrent: settings - .property("jmap.protocol.request.max-concurrent")? - .unwrap_or(4), - get_max_objects: settings - .property("jmap.protocol.get.max-objects")? - .unwrap_or(500), - set_max_objects: settings - .property("jmap.protocol.set.max-objects")? - .unwrap_or(500), - upload_max_size: settings - .property("jmap.protocol.upload.max-size")? - .unwrap_or(50000000), - upload_max_concurrent: settings - .property("jmap.protocol.upload.max-concurrent")? - .unwrap_or(4), - upload_tmp_quota_size: settings - .property("jmap.protocol.upload.quota.size")? - .unwrap_or(50000000), - upload_tmp_quota_amount: settings - .property("jmap.protocol.upload.quota.files")? - .unwrap_or(1000), - upload_tmp_ttl: settings - .property_or_default::("jmap.protocol.upload.ttl", "1h")? - .as_secs(), - mailbox_max_depth: settings.property("jmap.mailbox.max-depth")?.unwrap_or(10), - mailbox_name_max_len: settings - .property("jmap.mailbox.max-name-length")? - .unwrap_or(255), - mail_attachments_max_size: settings - .property("jmap.email.max-attachment-size")? - .unwrap_or(50000000), - mail_max_size: settings - .property("jmap.email.max-size")? - .unwrap_or(75000000), - mail_parse_max_items: settings - .property("jmap.email.parse.max-items")? - .unwrap_or(10), - sieve_max_script_name: settings - .property("sieve.untrusted.limits.name-length")? - .unwrap_or(512), - sieve_max_scripts: settings - .property("sieve.untrusted.limits.max-scripts")? - .unwrap_or(256), - capabilities: BaseCapabilities::default(), - session_cache_ttl: settings - .property("cache.session.ttl")? - .unwrap_or(Duration::from_secs(3600)), - rate_authenticated: settings - .property_or_default("jmap.rate-limit.account", "1000/1m")?, - rate_authenticate_req: settings - .property_or_default("authentication.rate-limit", "10/1m")?, - rate_anonymous: settings.property_or_default("jmap.rate-limit.anonymous", "100/1m")?, - rate_use_forwarded: settings - .property("jmap.rate-limit.use-forwarded")? - .unwrap_or(false), - oauth_key: settings - .value("oauth.key") - .map(|s| s.to_string()) - .unwrap_or_else(|| { - thread_rng() - .sample_iter(Alphanumeric) - .take(64) - .map(char::from) - .collect::() - }), - oauth_expiry_user_code: settings - .property_or_default::("oauth.expiry.user-code", "30m")? - .as_secs(), - oauth_expiry_auth_code: settings - .property_or_default::("oauth.expiry.auth-code", "10m")? - .as_secs(), - oauth_expiry_token: settings - .property_or_default::("oauth.expiry.token", "1h")? - .as_secs(), - oauth_expiry_refresh_token: settings - .property_or_default::("oauth.expiry.refresh-token", "30d")? - .as_secs(), - oauth_expiry_refresh_token_renew: settings - .property_or_default::("oauth.expiry.refresh-token-renew", "4d")? - .as_secs(), - oauth_max_auth_attempts: settings - .property_or_default("oauth.auth.max-attempts", "3")?, - event_source_throttle: settings - .property_or_default("jmap.event-source.throttle", "1s")?, - web_socket_throttle: settings.property_or_default("jmap.web-socket.throttle", "1s")?, - web_socket_timeout: settings.property_or_default("jmap.web-socket.timeout", "10m")?, - web_socket_heartbeat: settings - .property_or_default("jmap.web-socket.heartbeat", "1m")?, - push_max_total: settings.property_or_default("jmap.push.max-total", "100")?, - principal_allow_lookups: settings - .property("jmap.principal.allow-lookups")? - .unwrap_or(true), - encrypt: settings.property_or_default("storage.encryption.enable", "true")?, - encrypt_append: settings.property_or_default("storage.encryption.append", "false")?, - spam_header: settings.value("spam.header.is-spam").and_then(|v| { - v.split_once(':').map(|(k, v)| { - ( - mail_parser::HeaderName::parse(k.trim().to_string()).unwrap(), - v.trim().to_string(), - ) - }) - }), - http_headers: settings - .values("server.http.headers") - .map(|(_, v)| { - if let Some((k, v)) = v.split_once(':') { - Ok(( - hyper::header::HeaderName::from_str(k.trim()).map_err(|err| { - format!( - "Invalid header found in property \"server.http.headers\": {}", - err - ) - })?, - hyper::header::HeaderValue::from_str(v.trim()).map_err(|err| { - format!( - "Invalid header found in property \"server.http.headers\": {}", - err - ) - })?, - )) - } else { - Err(format!( - "Invalid header found in property \"server.http.headers\": {}", - v - )) - } - }) - .collect::, String>>()?, - }; - config.add_capabilites(settings); - Ok(config) - } -} diff --git a/crates/jmap/src/api/event_source.rs b/crates/jmap/src/api/event_source.rs index 929abb74..0c1e6ec8 100644 --- a/crates/jmap/src/api/event_source.rs +++ b/crates/jmap/src/api/event_source.rs @@ -106,7 +106,7 @@ impl JMAP { None }; let mut response = StateChangeResponse::new(); - let throttle = self.config.event_source_throttle; + let throttle = self.core.jmap.event_source_throttle; // Register with state manager let mut change_rx = if let Some(change_rx) = self diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 0eaee018..c53aed0a 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -23,6 +23,12 @@ use std::{net::IpAddr, sync::Arc}; +use common::{ + config::smtp::{V_LISTENER, V_LOCAL_IP, V_REMOTE_IP}, + expr::functions::ResolveVariable, + listener::{ServerInstance, SessionData, SessionManager, SessionStream}, + Core, +}; use http_body_util::{BodyExt, Full}; use hyper::{ body::{self, Bytes}, @@ -34,214 +40,240 @@ use hyper::{ use hyper_util::rt::TokioIo; use jmap_proto::{ error::request::{RequestError, RequestLimitError}, - request::Request, + request::{capability::Session, Request}, response::Response, types::{blob::BlobId, id::Id}, }; -use utils::listener::{ServerInstance, SessionData, SessionManager, SessionStream}; - use crate::{ auth::{oauth::OAuthMetadata, AccessToken}, blob::{DownloadResponse, UploadResponse}, services::state, - websocket::upgrade::upgrade_websocket_connection, JMAP, }; -use super::{ - session::Session, HtmlResponse, HttpRequest, HttpResponse, JmapSessionManager, JsonResponse, -}; +use super::{HtmlResponse, HttpRequest, HttpResponse, JmapSessionManager, JsonResponse}; -pub async fn parse_jmap_request( - jmap: Arc, - mut req: HttpRequest, - remote_ip: IpAddr, - instance: Arc, -) -> HttpResponse { - let mut path = req.uri().path().split('/'); - path.next(); +pub struct HttpSessionData { + pub instance: Arc, + pub local_ip: IpAddr, + pub remote_ip: IpAddr, + pub is_tls: bool, +} - match path.next().unwrap_or("") { - "jmap" => { - // Authenticate request - let (_in_flight, access_token) = match jmap.authenticate_headers(&req, remote_ip).await - { - Ok(Some(session)) => session, - Ok(None) => { - return if req.method() != Method::OPTIONS { - RequestError::unauthorized().into_http_response() - } else { - ().into_http_response() - } - } - Err(err) => return err.into_http_response(), - }; +impl JMAP { + pub async fn parse_http_request( + &self, + mut req: HttpRequest, + session: HttpSessionData, + ) -> HttpResponse { + let mut path = req.uri().path().split('/'); + path.next(); - match (path.next().unwrap_or(""), req.method()) { - ("", &Method::POST) => { - return match fetch_body(&mut req, jmap.config.request_max_size, &access_token) + match path.next().unwrap_or("") { + "jmap" => { + // Authenticate request + let (_in_flight, access_token) = + match self.authenticate_headers(&req, session.remote_ip).await { + Ok(Some(session)) => session, + Ok(None) => { + return if req.method() != Method::OPTIONS { + RequestError::unauthorized().into_http_response() + } else { + ().into_http_response() + } + } + Err(err) => return err.into_http_response(), + }; + + match (path.next().unwrap_or(""), req.method()) { + ("", &Method::POST) => { + return match fetch_body( + &mut req, + self.core.jmap.request_max_size, + &access_token, + ) .await .ok_or_else(|| RequestError::limit(RequestLimitError::SizeRequest)) .and_then(|bytes| { Request::parse( &bytes, - jmap.config.request_max_calls, - jmap.config.request_max_size, + self.core.jmap.request_max_calls, + self.core.jmap.request_max_size, ) }) { - Ok(request) => { - //let _ = println!("<- {}", String::from_utf8_lossy(&bytes)); + Ok(request) => { + //let _ = println!("<- {}", String::from_utf8_lossy(&bytes)); - match jmap.handle_request(request, access_token, &instance).await { - Ok(response) => response.into_http_response(), - Err(err) => err.into_http_response(), - } - } - Err(err) => err.into_http_response(), - }; - } - ("download", &Method::GET) => { - if let (Some(_), Some(blob_id), Some(name)) = ( - path.next().and_then(|p| Id::from_bytes(p.as_bytes())), - path.next().and_then(BlobId::from_base32), - path.next(), - ) { - return match jmap.blob_download(&blob_id, &access_token).await { - Ok(Some(blob)) => DownloadResponse { - filename: name.to_string(), - content_type: req - .uri() - .query() - .and_then(|q| { - form_urlencoded::parse(q.as_bytes()) - .find(|(k, _)| k == "accept") - .map(|(_, v)| v.into_owned()) - }) - .unwrap_or("application/octet-stream".to_string()), - blob, - } - .into_http_response(), - Ok(None) => RequestError::not_found().into_http_response(), - Err(_) => RequestError::internal_server_error().into_http_response(), - }; - } - } - ("upload", &Method::POST) => { - if let Some(account_id) = path.next().and_then(|p| Id::from_bytes(p.as_bytes())) - { - return match fetch_body( - &mut req, - jmap.config.upload_max_size, - &access_token, - ) - .await - { - Some(bytes) => { - match jmap - .blob_upload( - account_id, - req.headers() - .get(CONTENT_TYPE) - .and_then(|h| h.to_str().ok()) - .unwrap_or("application/octet-stream"), - &bytes, - access_token, - ) + match self + .handle_request(request, access_token, &session.instance) .await { Ok(response) => response.into_http_response(), Err(err) => err.into_http_response(), } } - None => RequestError::limit(RequestLimitError::SizeUpload) - .into_http_response(), + Err(err) => err.into_http_response(), }; } + ("download", &Method::GET) => { + if let (Some(_), Some(blob_id), Some(name)) = ( + path.next().and_then(|p| Id::from_bytes(p.as_bytes())), + path.next().and_then(BlobId::from_base32), + path.next(), + ) { + return match self.blob_download(&blob_id, &access_token).await { + Ok(Some(blob)) => DownloadResponse { + filename: name.to_string(), + content_type: req + .uri() + .query() + .and_then(|q| { + form_urlencoded::parse(q.as_bytes()) + .find(|(k, _)| k == "accept") + .map(|(_, v)| v.into_owned()) + }) + .unwrap_or("application/octet-stream".to_string()), + blob, + } + .into_http_response(), + Ok(None) => RequestError::not_found().into_http_response(), + Err(_) => { + RequestError::internal_server_error().into_http_response() + } + }; + } + } + ("upload", &Method::POST) => { + if let Some(account_id) = + path.next().and_then(|p| Id::from_bytes(p.as_bytes())) + { + return match fetch_body( + &mut req, + self.core.jmap.upload_max_size, + &access_token, + ) + .await + { + Some(bytes) => { + match self + .blob_upload( + account_id, + req.headers() + .get(CONTENT_TYPE) + .and_then(|h| h.to_str().ok()) + .unwrap_or("application/octet-stream"), + &bytes, + access_token, + ) + .await + { + Ok(response) => response.into_http_response(), + Err(err) => err.into_http_response(), + } + } + None => RequestError::limit(RequestLimitError::SizeUpload) + .into_http_response(), + }; + } + } + ("eventsource", &Method::GET) => { + return self.handle_event_source(req, access_token).await + } + ("ws", &Method::GET) => { + return self + .upgrade_websocket_connection( + req, + access_token, + session.instance.clone(), + ) + .await; + } + (_, &Method::OPTIONS) => { + return ().into_http_response(); + } + _ => (), } - ("eventsource", &Method::GET) => { - return jmap.handle_event_source(req, access_token).await + } + ".well-known" => match (path.next().unwrap_or(""), req.method()) { + ("jmap", &Method::GET) => { + // Authenticate request + let (_in_flight, access_token) = + match self.authenticate_headers(&req, session.remote_ip).await { + Ok(Some(session)) => session, + Ok(None) => return RequestError::unauthorized().into_http_response(), + Err(err) => return err.into_http_response(), + }; + + return match self + .handle_session_resource( + session.resolve_url(&self.core).await, + access_token, + ) + .await + { + Ok(session) => session.into_http_response(), + Err(err) => err.into_http_response(), + }; } - ("ws", &Method::GET) => { - return upgrade_websocket_connection(jmap, req, access_token, instance.clone()) - .await; + ("oauth-authorization-server", &Method::GET) => { + // Limit anonymous requests + return match self.is_anonymous_allowed(&session.remote_ip).await { + Ok(_) => JsonResponse::new(OAuthMetadata::new( + session.resolve_url(&self.core).await, + )) + .into_http_response(), + Err(err) => err.into_http_response(), + }; } (_, &Method::OPTIONS) => { return ().into_http_response(); } _ => (), - } - } - ".well-known" => match (path.next().unwrap_or(""), req.method()) { - ("jmap", &Method::GET) => { - // Authenticate request - let (_in_flight, access_token) = - match jmap.authenticate_headers(&req, remote_ip).await { - Ok(Some(session)) => session, - Ok(None) => return RequestError::unauthorized().into_http_response(), - Err(err) => return err.into_http_response(), - }; - - return match jmap.handle_session_resource(instance, access_token).await { - Ok(session) => session.into_http_response(), - Err(err) => err.into_http_response(), - }; - } - ("oauth-authorization-server", &Method::GET) => { - let remote_addr = jmap.build_remote_addr(&req, remote_ip); - // Limit anonymous requests - return match jmap.is_anonymous_allowed(&remote_addr).await { - Ok(_) => { - JsonResponse::new(OAuthMetadata::new(&instance.data)).into_http_response() - } - Err(err) => err.into_http_response(), - }; - } - (_, &Method::OPTIONS) => { - return ().into_http_response(); - } - _ => (), - }, - "auth" => { - let remote_addr = jmap.build_remote_addr(&req, remote_ip); - - match (path.next().unwrap_or(""), req.method()) { + }, + "auth" => match (path.next().unwrap_or(""), req.method()) { ("", &Method::GET) => { - return match jmap.is_anonymous_allowed(&remote_addr).await { - Ok(_) => jmap.handle_user_device_auth(&mut req).await, + return match self.is_anonymous_allowed(&session.remote_ip).await { + Ok(_) => self.handle_user_device_auth(&mut req).await, Err(err) => err.into_http_response(), } } ("", &Method::POST) => { - return match jmap.is_auth_allowed_soft(&remote_addr).await { + return match self.is_auth_allowed_soft(&session.remote_ip).await { Ok(_) => { - jmap.handle_user_device_auth_post(&mut req, remote_addr) + self.handle_user_device_auth_post(&mut req, session.remote_ip) .await } Err(err) => err.into_http_response(), } } ("code", &Method::GET) => { - return match jmap.is_anonymous_allowed(&remote_addr).await { - Ok(_) => jmap.handle_user_code_auth(&mut req).await, + return match self.is_anonymous_allowed(&session.remote_ip).await { + Ok(_) => self.handle_user_code_auth(&mut req).await, Err(err) => err.into_http_response(), } } ("code", &Method::POST) => { - return match jmap.is_auth_allowed_soft(&remote_addr).await { - Ok(_) => jmap.handle_user_code_auth_post(&mut req, remote_addr).await, + return match self.is_auth_allowed_soft(&session.remote_ip).await { + Ok(_) => { + self.handle_user_code_auth_post(&mut req, session.remote_ip) + .await + } Err(err) => err.into_http_response(), } } ("device", &Method::POST) => { - return match jmap.is_anonymous_allowed(&remote_addr).await { - Ok(_) => jmap.handle_device_auth(&mut req, instance).await, + return match self.is_anonymous_allowed(&session.remote_ip).await { + Ok(_) => { + self.handle_device_auth(&mut req, session.resolve_url(&self.core).await) + .await + } Err(err) => err.into_http_response(), } } ("token", &Method::POST) => { - return match jmap.is_anonymous_allowed(&remote_addr).await { - Ok(_) => jmap.handle_token_request(&mut req).await, + return match self.is_anonymous_allowed(&session.remote_ip).await { + Ok(_) => self.handle_token_request(&mut req).await, Err(err) => err.into_http_response(), } } @@ -249,116 +281,160 @@ pub async fn parse_jmap_request( return ().into_http_response(); } _ => (), - } - } - "crypto" if jmap.config.encrypt => { - let remote_addr = jmap.build_remote_addr(&req, remote_ip); - - match *req.method() { + }, + "crypto" if self.core.jmap.encrypt => match *req.method() { Method::GET => { - return jmap.handle_crypto_update(&mut req, remote_addr).await; + return self.handle_crypto_update(&mut req, session.remote_ip).await; } Method::POST => { - return match jmap.is_auth_allowed_soft(&remote_addr).await { - Ok(_) => jmap.handle_crypto_update(&mut req, remote_addr).await, + return match self.is_auth_allowed_soft(&session.remote_ip).await { + Ok(_) => self.handle_crypto_update(&mut req, session.remote_ip).await, Err(err) => err.into_http_response(), } } _ => (), - } - } - "api" => { - // Allow CORS preflight requests - if req.method() == Method::OPTIONS { - return ().into_http_response(); - } - - // Make sure the user is a superuser - return match jmap.authenticate_headers(&req, remote_ip).await { - Ok(Some((_, access_token))) => { - let body = fetch_body(&mut req, 8192, &access_token).await; - if access_token.is_super_user() { - jmap.handle_api_manage_request(&req, body, access_token) - .await - } else { - jmap.handle_api_request(&req, body, access_token).await - } + }, + "api" => { + // Allow CORS preflight requests + if req.method() == Method::OPTIONS { + return ().into_http_response(); } - Ok(None) => RequestError::unauthorized().into_http_response(), - Err(err) => err.into_http_response(), - }; + + // Make sure the user is a superuser + return match self.authenticate_headers(&req, session.remote_ip).await { + Ok(Some((_, access_token))) => { + let body = fetch_body(&mut req, 8192, &access_token).await; + if access_token.is_super_user() { + self.handle_api_manage_request(&req, body, access_token) + .await + } else { + self.handle_api_request(&req, body, access_token).await + } + } + Ok(None) => RequestError::unauthorized().into_http_response(), + Err(err) => err.into_http_response(), + }; + } + _ => (), + } + RequestError::not_found().into_http_response() + } + + async fn handle_session(self, session: SessionData) { + let span = session.span; + let _in_flight = session.in_flight; + let is_tls = session.stream.is_tls(); + + if let Err(http_err) = http1::Builder::new() + .keep_alive(true) + .serve_connection( + TokioIo::new(session.stream), + service_fn(|req: hyper::Request| { + let jmap = self.clone(); + let span = span.clone(); + let instance = session.instance.clone(); + + async move { + tracing::debug!( + parent: &span, + event = "request", + uri = req.uri().to_string(), + ); + + // Obtain remote IP + let remote_ip = if !jmap.core.jmap.http_use_forwarded { + session.remote_ip + } else if let Some(forwarded_for) = req + .headers() + .get(header::FORWARDED) + .or_else(|| req.headers().get("X-Forwarded-For")) + .and_then(|h| h.to_str().ok()) + .and_then(|h| h.parse::().ok()) + { + forwarded_for + } else { + tracing::warn!( + "Warning: No remote address found in request, using remote ip." + ); + session.remote_ip + }; + + // Parse HTTP request + let mut response = jmap + .parse_http_request( + req, + HttpSessionData { + instance, + local_ip: session.local_ip, + remote_ip, + is_tls, + }, + ) + .await; + + // Add custom headers + if !jmap.core.jmap.http_headers.is_empty() { + let headers = response.headers_mut(); + + for (header, value) in &jmap.core.jmap.http_headers { + headers.insert(header.clone(), value.clone()); + } + } + + Ok::<_, hyper::Error>(response) + } + }), + ) + .with_upgrades() + .await + { + tracing::debug!( + parent: &span, + event = "error", + context = "http", + reason = %http_err, + ); } - _ => (), } - RequestError::not_found().into_http_response() } impl SessionManager for JmapSessionManager { - fn handle( + fn handle( self, session: SessionData, ) -> impl std::future::Future + Send { - handle_request(self.inner, session) + JMAP::from(self.inner).handle_session(session) } #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async { - let _ = self.inner.state_tx.send(state::Event::Stop).await; + let _ = self + .inner + .jmap_inner + .state_tx + .send(state::Event::Stop) + .await; } } - - fn is_ip_blocked(&self, addr: &IpAddr) -> bool { - false - } } -async fn handle_request(jmap: Arc, session: SessionData) { - let span = session.span; - let _in_flight = session.in_flight; +impl ResolveVariable for HttpSessionData { + fn resolve_variable(&self, variable: u32) -> common::expr::Variable<'_> { + match variable { + V_REMOTE_IP => self.remote_ip.to_string().into(), + V_LOCAL_IP => self.local_ip.to_string().into(), + V_LISTENER => self.instance.id.as_str().into(), + _ => common::expr::Variable::default(), + } + } +} - if let Err(http_err) = http1::Builder::new() - .keep_alive(true) - .serve_connection( - TokioIo::new(session.stream), - service_fn(|req: hyper::Request| { - let jmap = jmap.clone(); - let span = span.clone(); - let instance = session.instance.clone(); - - async move { - tracing::debug!( - parent: &span, - event = "request", - uri = req.uri().to_string(), - ); - - // Parse JMAP request - let mut response = - parse_jmap_request(jmap.clone(), req, session.remote_ip, instance).await; - - // Add custom headers - if !jmap.config.http_headers.is_empty() { - let headers = response.headers_mut(); - - for (header, value) in &jmap.config.http_headers { - headers.insert(header.clone(), value.clone()); - } - } - - Ok::<_, hyper::Error>(response) - } - }), - ) - .with_upgrades() - .await - { - tracing::debug!( - parent: &span, - event = "error", - context = "http", - reason = %http_err, - ); +impl HttpSessionData { + pub async fn resolve_url(&self, core: &Core) -> String { + core.eval_if(&core.network.url, self) + .await + .unwrap_or_else(|| format!("http{}://localhost", if self.is_tls { "s" } else { "" })) } } diff --git a/crates/jmap/src/api/mod.rs b/crates/jmap/src/api/mod.rs index 927225d6..5916dca0 100644 --- a/crates/jmap/src/api/mod.rs +++ b/crates/jmap/src/api/mod.rs @@ -21,17 +21,14 @@ * for more details. */ -use std::sync::Arc; - use hyper::StatusCode; use jmap_proto::types::{id::Id, state::State, type_state::DataType}; use serde::Serialize; use utils::map::vec_map::VecMap; -use crate::JMAP; +use crate::JmapInstance; pub mod admin; -pub mod config; pub mod event_source; pub mod http; pub mod request; @@ -39,11 +36,11 @@ pub mod session; #[derive(Clone)] pub struct JmapSessionManager { - pub inner: Arc, + pub inner: JmapInstance, } impl JmapSessionManager { - pub fn new(inner: Arc) -> Self { + pub fn new(inner: JmapInstance) -> Self { Self { inner } } } diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index e817e2db..0ebe2fcf 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use common::listener::ServerInstance; use jmap_proto::{ error::{method::MethodError, request::RequestError}, method::{ @@ -33,7 +34,6 @@ use jmap_proto::{ response::{Response, ResponseMethod}, types::collection::Collection, }; -use utils::listener::ServerInstance; use crate::{auth::AccessToken, JMAP}; @@ -173,7 +173,7 @@ impl JMAP { self.vacation_response_get(req).await?.into() } get::RequestArguments::Principal => { - if self.config.principal_allow_lookups || access_token.is_super_user() { + if self.core.jmap.principal_allow_lookups || access_token.is_super_user() { self.principal_get(req).await?.into() } else { return Err(MethodError::Forbidden( @@ -220,7 +220,7 @@ impl JMAP { self.sieve_script_query(req).await?.into() } query::RequestArguments::Principal => { - if self.config.principal_allow_lookups || access_token.is_super_user() { + if self.core.jmap.principal_allow_lookups || access_token.is_super_user() { self.principal_query(req).await?.into() } else { return Err(MethodError::Forbidden( diff --git a/crates/jmap/src/api/session.rs b/crates/jmap/src/api/session.rs index de72df7a..5032b661 100644 --- a/crates/jmap/src/api/session.rs +++ b/crates/jmap/src/api/session.rs @@ -26,170 +26,19 @@ use std::sync::Arc; use directory::QueryBy; use jmap_proto::{ error::request::RequestError, - request::capability::Capability, - response::serialize::serialize_hex, - types::{acl::Acl, collection::Collection, id::Id, type_state::DataType}, + request::capability::{Capability, Session}, + types::{acl::Acl, collection::Collection, id::Id}, }; -use store::ahash::AHashSet; -use utils::{listener::ServerInstance, map::vec_map::VecMap, UnwrapFailure}; use crate::{auth::AccessToken, JMAP}; -#[derive(Debug, Clone, serde::Serialize)] -pub struct Session { - #[serde(rename(serialize = "capabilities"))] - capabilities: VecMap, - #[serde(rename(serialize = "accounts"))] - accounts: VecMap, - #[serde(rename(serialize = "primaryAccounts"))] - primary_accounts: VecMap, - #[serde(rename(serialize = "username"))] - username: String, - #[serde(rename(serialize = "apiUrl"))] - api_url: String, - #[serde(rename(serialize = "downloadUrl"))] - download_url: String, - #[serde(rename(serialize = "uploadUrl"))] - upload_url: String, - #[serde(rename(serialize = "eventSourceUrl"))] - event_source_url: String, - #[serde(rename(serialize = "state"))] - #[serde(serialize_with = "serialize_hex")] - state: u32, - #[serde(skip)] - base_url: String, -} - -#[derive(Debug, Clone, serde::Serialize)] -struct Account { - #[serde(rename(serialize = "name"))] - name: String, - #[serde(rename(serialize = "isPersonal"))] - is_personal: bool, - #[serde(rename(serialize = "isReadOnly"))] - is_read_only: bool, - #[serde(rename(serialize = "accountCapabilities"))] - account_capabilities: VecMap, -} - -#[derive(Debug, Clone, serde::Serialize)] -#[serde(untagged)] -#[allow(dead_code)] -pub enum Capabilities { - Core(CoreCapabilities), - Mail(MailCapabilities), - Submission(SubmissionCapabilities), - WebSocket(WebSocketCapabilities), - SieveAccount(SieveAccountCapabilities), - SieveSession(SieveSessionCapabilities), - Blob(BlobCapabilities), - Empty(EmptyCapabilities), -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct CoreCapabilities { - #[serde(rename(serialize = "maxSizeUpload"))] - max_size_upload: usize, - #[serde(rename(serialize = "maxConcurrentUpload"))] - max_concurrent_upload: usize, - #[serde(rename(serialize = "maxSizeRequest"))] - max_size_request: usize, - #[serde(rename(serialize = "maxConcurrentRequests"))] - max_concurrent_requests: usize, - #[serde(rename(serialize = "maxCallsInRequest"))] - max_calls_in_request: usize, - #[serde(rename(serialize = "maxObjectsInGet"))] - max_objects_in_get: usize, - #[serde(rename(serialize = "maxObjectsInSet"))] - max_objects_in_set: usize, - #[serde(rename(serialize = "collationAlgorithms"))] - collation_algorithms: Vec, -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct WebSocketCapabilities { - #[serde(rename(serialize = "url"))] - url: String, - #[serde(rename(serialize = "supportsPush"))] - supports_push: bool, -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct SieveSessionCapabilities { - #[serde(rename(serialize = "implementation"))] - pub implementation: &'static str, -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct SieveAccountCapabilities { - #[serde(rename(serialize = "maxSizeScriptName"))] - pub max_script_name: usize, - #[serde(rename(serialize = "maxSizeScript"))] - pub max_script_size: usize, - #[serde(rename(serialize = "maxNumberScripts"))] - pub max_scripts: usize, - #[serde(rename(serialize = "maxNumberRedirects"))] - pub max_redirects: usize, - #[serde(rename(serialize = "sieveExtensions"))] - pub extensions: Vec, - #[serde(rename(serialize = "notificationMethods"))] - pub notification_methods: Option>, - #[serde(rename(serialize = "externalLists"))] - pub ext_lists: Option>, -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct MailCapabilities { - #[serde(rename(serialize = "maxMailboxesPerEmail"))] - max_mailboxes_per_email: Option, - #[serde(rename(serialize = "maxMailboxDepth"))] - max_mailbox_depth: usize, - #[serde(rename(serialize = "maxSizeMailboxName"))] - max_size_mailbox_name: usize, - #[serde(rename(serialize = "maxSizeAttachmentsPerEmail"))] - max_size_attachments_per_email: usize, - #[serde(rename(serialize = "emailQuerySortOptions"))] - email_query_sort_options: Vec, - #[serde(rename(serialize = "mayCreateTopLevelMailbox"))] - may_create_top_level_mailbox: bool, -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct SubmissionCapabilities { - #[serde(rename(serialize = "maxDelayedSend"))] - max_delayed_send: usize, - #[serde(rename(serialize = "submissionExtensions"))] - submission_extensions: VecMap>, -} - -#[derive(Debug, Clone, serde::Serialize)] -pub struct BlobCapabilities { - #[serde(rename(serialize = "maxSizeBlobSet"))] - max_size_blob_set: usize, - #[serde(rename(serialize = "maxDataSources"))] - max_data_sources: usize, - #[serde(rename(serialize = "supportedTypeNames"))] - supported_type_names: Vec, - #[serde(rename(serialize = "supportedDigestAlgorithms"))] - supported_digest_algorithms: Vec<&'static str>, -} - -#[derive(Debug, Clone, Default, serde::Serialize)] -pub struct EmptyCapabilities {} - -#[derive(Default)] -pub struct BaseCapabilities { - pub session: VecMap, - pub account: VecMap, -} - impl JMAP { pub async fn handle_session_resource( &self, - instance: Arc, + base_url: String, access_token: Arc, ) -> Result { - let mut session = Session::new(&instance.data, &self.config.capabilities); + let mut session = Session::new(base_url, &self.core.jmap.capabilities); session.set_state(access_token.state()); session.set_primary_account( access_token.primary_id().into(), @@ -199,7 +48,7 @@ impl JMAP { .clone() .unwrap_or_else(|| access_token.name.clone()), None, - &self.config.capabilities.account, + &self.core.jmap.capabilities.account, ); // Add secondary accounts @@ -213,7 +62,9 @@ impl JMAP { session.add_account( (*id).into(), - self.directory + self.core + .storage + .directory .query(QueryBy::Id(*id), false) .await .unwrap_or_default() @@ -222,320 +73,10 @@ impl JMAP { is_personal, is_readonly, Some(&[Capability::Mail, Capability::Quota, Capability::Blob]), - &self.config.capabilities.account, + &self.core.jmap.capabilities.account, ); } Ok(session) } } - -impl crate::Config { - pub fn add_capabilites(&mut self, settings: &utils::config::Config) { - // Add core capabilities - self.capabilities.session.append( - Capability::Core, - Capabilities::Core(CoreCapabilities::new(self)), - ); - - // Add email capabilities - self.capabilities.session.append( - Capability::Mail, - Capabilities::Empty(EmptyCapabilities::default()), - ); - self.capabilities.account.append( - Capability::Mail, - Capabilities::Mail(MailCapabilities::new(self)), - ); - - // Add submission capabilities - self.capabilities.session.append( - Capability::Submission, - Capabilities::Empty(EmptyCapabilities::default()), - ); - self.capabilities.account.append( - Capability::Submission, - Capabilities::Submission(SubmissionCapabilities { - max_delayed_send: 86400 * 30, - submission_extensions: VecMap::from_iter([ - ("FUTURERELEASE".to_string(), Vec::new()), - ("SIZE".to_string(), Vec::new()), - ("DSN".to_string(), Vec::new()), - ("DELIVERYBY".to_string(), Vec::new()), - ("MT-PRIORITY".to_string(), vec!["MIXER".to_string()]), - ("REQUIRETLS".to_string(), vec![]), - ]), - }), - ); - - // Add vacation response capabilities - self.capabilities.session.append( - Capability::VacationResponse, - Capabilities::Empty(EmptyCapabilities::default()), - ); - self.capabilities.account.append( - Capability::VacationResponse, - Capabilities::Empty(EmptyCapabilities::default()), - ); - - // Add Sieve capabilities - self.capabilities.session.append( - Capability::Sieve, - Capabilities::SieveSession(SieveSessionCapabilities::default()), - ); - self.capabilities.account.append( - Capability::Sieve, - Capabilities::SieveAccount(SieveAccountCapabilities::new(self, settings)), - ); - - // Add Blob capabilities - self.capabilities.session.append( - Capability::Blob, - Capabilities::Empty(EmptyCapabilities::default()), - ); - self.capabilities.account.append( - Capability::Blob, - Capabilities::Blob(BlobCapabilities::new(self)), - ); - - // Add Quota capabilities - self.capabilities.session.append( - Capability::Quota, - Capabilities::Empty(EmptyCapabilities::default()), - ); - self.capabilities.account.append( - Capability::Quota, - Capabilities::Empty(EmptyCapabilities::default()), - ); - } -} - -impl Session { - pub fn new(base_url: &str, base_capabilities: &BaseCapabilities) -> Session { - let mut capabilities = base_capabilities.session.clone(); - capabilities.append( - Capability::WebSocket, - Capabilities::WebSocket(WebSocketCapabilities::new(base_url)), - ); - - Session { - capabilities, - accounts: VecMap::new(), - primary_accounts: VecMap::new(), - username: "".to_string(), - api_url: format!("{}/jmap/", base_url), - download_url: format!( - "{}/jmap/download/{{accountId}}/{{blobId}}/{{name}}?accept={{type}}", - base_url - ), - upload_url: format!("{}/jmap/upload/{{accountId}}/", base_url), - event_source_url: format!( - "{}/jmap/eventsource/?types={{types}}&closeafter={{closeafter}}&ping={{ping}}", - base_url - ), - base_url: base_url.to_string(), - state: 0, - } - } - - pub fn set_primary_account( - &mut self, - account_id: Id, - username: String, - name: String, - capabilities: Option<&[Capability]>, - account_capabilities: &VecMap, - ) { - self.username = username; - - if let Some(capabilities) = capabilities { - for capability in capabilities { - self.primary_accounts.append(*capability, account_id); - } - } else { - for capability in self.capabilities.keys() { - self.primary_accounts.append(*capability, account_id); - } - } - - self.accounts.set( - account_id, - Account::new(name, true, false).add_capabilities(capabilities, account_capabilities), - ); - } - - pub fn add_account( - &mut self, - account_id: Id, - name: String, - is_personal: bool, - is_read_only: bool, - capabilities: Option<&[Capability]>, - account_capabilities: &VecMap, - ) { - self.accounts.set( - account_id, - Account::new(name, is_personal, is_read_only) - .add_capabilities(capabilities, account_capabilities), - ); - } - - pub fn set_state(&mut self, state: u32) { - self.state = state; - } - - pub fn api_url(&self) -> &str { - &self.api_url - } - - pub fn base_url(&self) -> &str { - &self.base_url - } -} - -impl Account { - pub fn new(name: String, is_personal: bool, is_read_only: bool) -> Account { - Account { - name, - is_personal, - is_read_only, - account_capabilities: VecMap::new(), - } - } - - pub fn add_capabilities( - mut self, - capabilities: Option<&[Capability]>, - account_capabilities: &VecMap, - ) -> Account { - if let Some(capabilities) = capabilities { - for capability in capabilities { - if let Some(value) = account_capabilities.get(capability) { - self.account_capabilities.append(*capability, value.clone()); - } - } - } else { - self.account_capabilities = account_capabilities.clone(); - } - self - } -} - -impl CoreCapabilities { - pub fn new(config: &crate::Config) -> Self { - CoreCapabilities { - max_size_upload: config.upload_max_size, - max_concurrent_upload: config.upload_max_concurrent as usize, - max_size_request: config.request_max_size, - max_concurrent_requests: config.request_max_concurrent as usize, - max_calls_in_request: config.request_max_calls, - max_objects_in_get: config.get_max_objects, - max_objects_in_set: config.set_max_objects, - collation_algorithms: vec![ - "i;ascii-numeric".to_string(), - "i;ascii-casemap".to_string(), - "i;unicode-casemap".to_string(), - ], - } - } -} - -impl WebSocketCapabilities { - pub fn new(base_url: &str) -> Self { - WebSocketCapabilities { - url: format!("ws{}/jmap/ws", base_url.strip_prefix("http").unwrap()), - supports_push: true, - } - } -} - -impl SieveAccountCapabilities { - pub fn new(config: &crate::Config, settings: &utils::config::Config) -> Self { - let mut notification_methods = Vec::new(); - - for (_, uri) in settings.values("sieve.untrusted.notification-uris") { - notification_methods.push(uri.to_string()); - } - if notification_methods.is_empty() { - notification_methods.push("mailto".to_string()); - } - - let mut capabilities: AHashSet = - AHashSet::from_iter(sieve::compiler::grammar::Capability::all().iter().cloned()); - - for (_, capability) in settings.values("sieve.untrusted.disabled-capabilities") { - capabilities.remove(&sieve::compiler::grammar::Capability::parse(capability)); - } - - let mut extensions = capabilities - .into_iter() - .map(|c| c.to_string()) - .collect::>(); - extensions.sort_unstable(); - - SieveAccountCapabilities { - max_script_name: config.sieve_max_script_name, - max_script_size: settings - .property("sieve.untrusted.max-script-size") - .failed("Invalid configuration file") - .unwrap_or(1024 * 1024), - max_scripts: config.sieve_max_scripts, - max_redirects: settings - .property("sieve.untrusted.max-redirects") - .failed("Invalid configuration file") - .unwrap_or(1), - extensions, - notification_methods: if !notification_methods.is_empty() { - notification_methods.into() - } else { - None - }, - ext_lists: None, - } - } -} - -impl Default for SieveSessionCapabilities { - fn default() -> Self { - Self { - implementation: concat!("Stalwart JMAP v", env!("CARGO_PKG_VERSION"),), - } - } -} - -impl MailCapabilities { - pub fn new(config: &crate::Config) -> Self { - MailCapabilities { - max_mailboxes_per_email: None, - max_mailbox_depth: config.mailbox_max_depth, - max_size_mailbox_name: config.mailbox_name_max_len, - max_size_attachments_per_email: config.mail_attachments_max_size, - email_query_sort_options: [ - "receivedAt", - "size", - "from", - "to", - "subject", - "sentAt", - "hasKeyword", - "allInThreadHaveKeyword", - "someInThreadHaveKeyword", - ] - .iter() - .map(|s| s.to_string()) - .collect(), - may_create_top_level_mailbox: true, - } - } -} - -impl BlobCapabilities { - pub fn new(config: &crate::Config) -> Self { - BlobCapabilities { - max_size_blob_set: (config.request_max_size * 3 / 4) - 512, - max_data_sources: config.request_max_calls, - supported_type_names: vec![DataType::Email, DataType::Thread, DataType::SieveScript], - supported_digest_algorithms: vec!["sha", "sha-256", "sha-512"], - } - } -} diff --git a/crates/jmap/src/auth/acl.rs b/crates/jmap/src/auth/acl.rs index edccb84e..a9546903 100644 --- a/crates/jmap/src/auth/acl.rs +++ b/crates/jmap/src/auth/acl.rs @@ -51,7 +51,9 @@ impl JMAP { .chain(access_token.member_of.clone().iter()) { for acl_item in self - .store + .core + .storage + .data .acl_query(AclQuery::HasAccess { grant_account_id }) .await .map_err(|err| { @@ -120,7 +122,9 @@ impl JMAP { .chain(access_token.member_of.clone().iter()) { for acl_item in self - .store + .core + .storage + .data .acl_query(AclQuery::SharedWith { grant_account_id, to_account_id, @@ -233,7 +237,9 @@ impl JMAP { .chain(access_token.member_of.clone().iter()) { match self - .store + .core + .storage + .data .get_value::(ValueKey { account_id: to_account_id, collection: to_collection, @@ -350,6 +356,8 @@ impl JMAP { let mut acl_obj = Object::with_capacity(value.len() / 2); for item in value { if let Some(principal) = self + .core + .storage .directory .query(QueryBy::Id(item.account_id), false) .await @@ -376,7 +384,7 @@ impl JMAP { current: &Option>>, ) { if let Value::Acl(acl_changes) = changes.get(&Property::Acl) { - let access_tokens = &self.access_tokens; + let access_tokens = &self.inner.access_tokens; if let Some(Value::Acl(acl_current)) = current .as_ref() .and_then(|current| current.inner.properties.get(&Property::Acl)) @@ -419,6 +427,8 @@ impl JMAP { for item in acl_set.chunks_exact(2) { if let (Value::Text(account_name), Value::UnsignedInt(grants)) = (&item[0], &item[1]) { match self + .core + .storage .directory .query(QueryBy::Name(account_name), false) .await @@ -458,6 +468,8 @@ impl JMAP { (&acl_patch[0], &acl_patch[1]) { match self + .core + .storage .directory .query(QueryBy::Name(account_name), false) .await diff --git a/crates/jmap/src/auth/authenticate.rs b/crates/jmap/src/auth/authenticate.rs index ff5dc921..f3494e17 100644 --- a/crates/jmap/src/auth/authenticate.rs +++ b/crates/jmap/src/auth/authenticate.rs @@ -23,12 +23,13 @@ use std::{net::IpAddr, sync::Arc, time::Instant}; +use common::{listener::limiter::InFlight, AuthResult}; use directory::QueryBy; use hyper::header; use jmap_proto::error::request::RequestError; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; -use utils::{listener::limiter::InFlight, map::ttl_dashmap::TtlMap}; +use utils::map::ttl_dashmap::TtlMap; use crate::JMAP; @@ -46,13 +47,12 @@ impl JMAP { .and_then(|h| h.to_str().ok()) .and_then(|h| h.split_once(' ').map(|(l, t)| (l, t.trim().to_string()))) { - let session = if let Some(account_id) = self.sessions.get_with_ttl(&token) { + let session = if let Some(account_id) = self.inner.sessions.get_with_ttl(&token) { self.get_cached_access_token(account_id).await } else { - let addr = self.build_remote_addr(req, remote_ip); if mechanism.eq_ignore_ascii_case("basic") { // Enforce rate limit for authentication requests - self.is_auth_allowed_soft(&addr).await?; + self.is_auth_allowed_soft(&remote_ip).await?; // Decode the base64 encoded credentials if let Some((account, secret)) = base64_decode(token.as_bytes()) @@ -64,7 +64,7 @@ impl JMAP { }) { if let AuthResult::Success(access_token) = - self.authenticate_plain(&account, &secret, addr).await + self.authenticate_plain(&account, &secret, remote_ip).await { Some(access_token) } else { @@ -80,7 +80,7 @@ impl JMAP { } } else if mechanism.eq_ignore_ascii_case("bearer") { // Enforce anonymous rate limit for bearer auth requests - self.is_anonymous_allowed(&addr).await?; + self.is_anonymous_allowed(&remote_ip).await?; match self.validate_access_token("access_token", &token).await { Ok((account_id, _, _)) => self.get_access_token(account_id).await, @@ -95,7 +95,7 @@ impl JMAP { } } else { // Enforce anonymous rate limit - self.is_anonymous_allowed(&addr).await?; + self.is_anonymous_allowed(&remote_ip).await?; None } .map(|access_token| { @@ -114,31 +114,30 @@ impl JMAP { } } else { // Enforce anonymous rate limit - self.is_anonymous_allowed(&self.build_remote_addr(req, remote_ip)) - .await?; + self.is_anonymous_allowed(&remote_ip).await?; Ok(None) } } pub fn cache_session(&self, session_id: String, access_token: &AccessToken) { - self.sessions.insert_with_ttl( + self.inner.sessions.insert_with_ttl( session_id, access_token.primary_id(), - Instant::now() + self.config.session_cache_ttl, + Instant::now() + self.core.jmap.session_cache_ttl, ); } pub fn cache_access_token(&self, access_token: Arc) { - self.access_tokens.insert_with_ttl( + self.inner.access_tokens.insert_with_ttl( access_token.primary_id(), access_token, - Instant::now() + self.config.session_cache_ttl, + Instant::now() + self.core.jmap.session_cache_ttl, ); } pub async fn get_cached_access_token(&self, primary_id: u32) -> Option> { - if let Some(access_token) = self.access_tokens.get_with_ttl(&primary_id) { + if let Some(access_token) = self.inner.access_tokens.get_with_ttl(&primary_id) { access_token.into() } else { // Refresh ACL token @@ -150,27 +149,6 @@ impl JMAP { } } - pub fn build_remote_addr( - &self, - req: &hyper::Request, - remote_ip: IpAddr, - ) -> IpAddr { - if !self.config.rate_use_forwarded { - remote_ip - } else if let Some(forwarded_for) = req - .headers() - .get(header::FORWARDED) - .or_else(|| req.headers().get("X-Forwarded-For")) - .and_then(|h| h.to_str().ok()) - .and_then(|h| h.parse::().ok()) - { - forwarded_for - } else { - tracing::warn!("Warning: No remote address found in request, using remote ip."); - remote_ip - } - } - pub async fn authenticate_plain( &self, username: &str, @@ -178,8 +156,9 @@ impl JMAP { remote_ip: IpAddr, ) -> AuthResult { match self - .directory + .core .authenticate( + &self.core.storage.directory, &Credentials::Plain { username: username.to_string(), secret: secret.to_string(), @@ -202,7 +181,9 @@ impl JMAP { pub async fn get_access_token(&self, account_id: u32) -> Option { // Create access token self.update_access_token(AccessToken::new( - self.directory + self.core + .storage + .directory .query(QueryBy::Id(account_id), true) .await .ok()??, diff --git a/crates/jmap/src/auth/oauth/device_auth.rs b/crates/jmap/src/auth/oauth/device_auth.rs index 9de460a0..4fb3a796 100644 --- a/crates/jmap/src/auth/oauth/device_auth.rs +++ b/crates/jmap/src/auth/oauth/device_auth.rs @@ -27,12 +27,13 @@ use std::{ time::{Duration, Instant}, }; +use common::AuthResult; use hyper::StatusCode; use store::rand::{ distributions::{Alphanumeric, Standard}, thread_rng, Rng, }; -use utils::{listener::ServerInstance, map::ttl_dashmap::TtlMap}; +use utils::map::ttl_dashmap::TtlMap; use crate::{ api::{http::ToHttpResponse, HtmlResponse, HttpRequest, HttpResponse, JsonResponse}, @@ -54,7 +55,7 @@ impl JMAP { pub async fn handle_device_auth( &self, req: &mut HttpRequest, - instance: Arc, + base_url: impl AsRef, ) -> HttpResponse { // Parse form let client_id = match FormData::from_request(req, MAX_POST_LEN) @@ -100,19 +101,22 @@ impl JMAP { client_id, redirect_uri: None, }); - let expiry = Instant::now() + Duration::from_secs(self.config.oauth_expiry_user_code); - self.oauth_codes + let expiry = Instant::now() + Duration::from_secs(self.core.jmap.oauth_expiry_user_code); + self.inner + .oauth_codes .insert_with_ttl(device_code.clone(), oauth_code.clone(), expiry); - self.oauth_codes + self.inner + .oauth_codes .insert_with_ttl(user_code.clone(), oauth_code, expiry); // Build response + let base_url = base_url.as_ref(); JsonResponse::new(DeviceAuthResponse { - verification_uri: format!("{}/auth", instance.data), - verification_uri_complete: format!("{}/auth/?code={}", instance.data, user_code), + verification_uri: format!("{}/auth", base_url), + verification_uri_complete: format!("{}/auth/?code={}", base_url, user_code), device_code, user_code, - expires_in: self.config.oauth_expiry_user_code, + expires_in: self.core.jmap.oauth_expiry_user_code, interval: 5, }) .into_http_response() @@ -168,9 +172,9 @@ impl JMAP { let code = if let Some(oauth) = fields .get("code") - .and_then(|code| self.oauth_codes.get_with_ttl(code)) + .and_then(|code| self.inner.oauth_codes.get_with_ttl(code)) { - if (STATUS_PENDING..STATUS_PENDING + self.config.oauth_max_auth_attempts) + if (STATUS_PENDING..STATUS_PENDING + self.core.jmap.oauth_max_auth_attempts) .contains(&oauth.status.load(atomic::Ordering::Relaxed)) { if let (Some(email), Some(password)) = (fields.get("email"), fields.get("password")) diff --git a/crates/jmap/src/auth/oauth/mod.rs b/crates/jmap/src/auth/oauth/mod.rs index f0632c79..962cb6b9 100644 --- a/crates/jmap/src/auth/oauth/mod.rs +++ b/crates/jmap/src/auth/oauth/mod.rs @@ -188,9 +188,10 @@ pub struct OAuthMetadata { } impl OAuthMetadata { - pub fn new(base_url: &str) -> Self { + pub fn new(base_url: impl AsRef) -> Self { + let base_url = base_url.as_ref(); OAuthMetadata { - issuer: base_url.to_string(), + issuer: base_url.into(), authorization_endpoint: format!("{}/auth/code", base_url), token_endpoint: format!("{}/auth/token", base_url), grant_types_supported: vec![ diff --git a/crates/jmap/src/auth/oauth/token.rs b/crates/jmap/src/auth/oauth/token.rs index deedc5d8..0e93cf2f 100644 --- a/crates/jmap/src/auth/oauth/token.rs +++ b/crates/jmap/src/auth/oauth/token.rs @@ -65,7 +65,7 @@ impl JMAP { params.get("client_id"), params.get("redirect_uri"), ) { - if let Some(oauth) = self.oauth_codes.get_with_ttl(code) { + if let Some(oauth) = self.inner.oauth_codes.get_with_ttl(code) { if client_id != oauth.client_id || redirect_uri != oauth.redirect_uri.as_deref().unwrap_or("") { @@ -103,7 +103,7 @@ impl JMAP { if let (Some(oauth), Some(client_id)) = ( params .get("device_code") - .and_then(|dc| self.oauth_codes.get_with_ttl(dc)), + .and_then(|dc| self.inner.oauth_codes.get_with_ttl(dc)), params.get("client_id"), ) { response = if oauth.client_id != client_id { @@ -131,7 +131,7 @@ impl JMAP { } status if (STATUS_PENDING - ..STATUS_PENDING + self.config.oauth_max_auth_attempts) + ..STATUS_PENDING + self.core.jmap.oauth_max_auth_attempts) .contains(&status) => { TokenResponse::error(ErrorType::AuthorizationPending) @@ -152,7 +152,7 @@ impl JMAP { .issue_token( account_id, &client_id, - time_left <= self.config.oauth_expiry_refresh_token_renew, + time_left <= self.core.jmap.oauth_expiry_refresh_token_renew, ) .await .map(TokenResponse::Granted) @@ -184,6 +184,8 @@ impl JMAP { with_refresh_token: bool, ) -> Result { let password_hash = self + .core + .storage .directory .query(QueryBy::Id(account_id), false) .await @@ -200,17 +202,17 @@ impl JMAP { account_id, &password_hash, client_id, - self.config.oauth_expiry_token, + self.core.jmap.oauth_expiry_token, )?, token_type: "bearer".to_string(), - expires_in: self.config.oauth_expiry_token, + expires_in: self.core.jmap.oauth_expiry_token, refresh_token: if with_refresh_token { self.encode_access_token( "refresh_token", account_id, &password_hash, client_id, - self.config.oauth_expiry_refresh_token, + self.core.jmap.oauth_expiry_refresh_token, )? .into() } else { @@ -232,7 +234,7 @@ impl JMAP { if client_id.len() > CLIENT_ID_MAX_LEN { return Err("ClientId is too long"); } - let key = self.config.oauth_key.clone(); + let key = self.core.jmap.oauth_key.clone(); let context = format!( "{} {} {} {}", grant_type, client_id, account_id, password_hash @@ -302,6 +304,8 @@ impl JMAP { // Obtain password hash let password_hash = self + .core + .storage .directory .query(QueryBy::Id(account_id), false) .await @@ -313,7 +317,7 @@ impl JMAP { .ok_or("Failed to obtain password hash")?; // Build context - let key = self.config.oauth_key.clone(); + let key = self.core.jmap.oauth_key.clone(); let context = format!( "{} {} {} {}", grant_type, client_id, account_id, password_hash diff --git a/crates/jmap/src/auth/oauth/user_code.rs b/crates/jmap/src/auth/oauth/user_code.rs index cdd6151a..d94fdc54 100644 --- a/crates/jmap/src/auth/oauth/user_code.rs +++ b/crates/jmap/src/auth/oauth/user_code.rs @@ -28,6 +28,7 @@ use std::{ time::{Duration, Instant}, }; +use common::AuthResult; use http_body_util::{BodyExt, Full}; use hyper::{body::Bytes, header, StatusCode}; use mail_builder::encoders::base64::base64_encode; @@ -122,7 +123,7 @@ impl JMAP { .collect::(); // Add client code - self.oauth_codes.insert_with_ttl( + self.inner.oauth_codes.insert_with_ttl( client_code.clone(), Arc::new(OAuthCode { status: STATUS_AUTHORIZED.into(), @@ -130,7 +131,7 @@ impl JMAP { client_id, redirect_uri, }), - Instant::now() + Duration::from_secs(self.config.oauth_expiry_auth_code), + Instant::now() + Duration::from_secs(self.core.jmap.oauth_expiry_auth_code), ); client_code } @@ -205,7 +206,7 @@ impl JMAP { let _ = write!(redirect_link, "&state={}", state); } - if auth_code.is_none() && (auth_attempts < self.config.oauth_max_auth_attempts) { + if auth_code.is_none() && (auth_attempts < self.core.jmap.oauth_max_auth_attempts) { let code = String::from_utf8( base64_encode( &bincode::serialize(&(auth_attempts + 1, code_req)).unwrap_or_default(), diff --git a/crates/jmap/src/auth/rate_limit.rs b/crates/jmap/src/auth/rate_limit.rs index 359c978a..3d10accc 100644 --- a/crates/jmap/src/auth/rate_limit.rs +++ b/crates/jmap/src/auth/rate_limit.rs @@ -23,8 +23,8 @@ use std::{net::IpAddr, sync::Arc}; +use common::listener::limiter::{ConcurrencyLimiter, InFlight}; use jmap_proto::error::request::{RequestError, RequestLimitError}; -use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; use crate::JMAP; @@ -37,17 +37,22 @@ pub struct ConcurrencyLimiters { impl JMAP { pub fn get_concurrency_limiter(&self, account_id: u32) -> Arc { - self.concurrency_limiter + self.inner + .concurrency_limiter .get(&account_id) .map(|limiter| limiter.clone()) .unwrap_or_else(|| { let limiter = Arc::new(ConcurrencyLimiters { concurrent_requests: ConcurrencyLimiter::new( - self.config.request_max_concurrent, + self.core.jmap.request_max_concurrent, + ), + concurrent_uploads: ConcurrencyLimiter::new( + self.core.jmap.upload_max_concurrent, ), - concurrent_uploads: ConcurrencyLimiter::new(self.config.upload_max_concurrent), }); - self.concurrency_limiter.insert(account_id, limiter.clone()); + self.inner + .concurrency_limiter + .insert(account_id, limiter.clone()); limiter }) } @@ -57,18 +62,23 @@ impl JMAP { access_token: &AccessToken, ) -> Result { let limiter = self.get_concurrency_limiter(access_token.primary_id()); + let is_rate_allowed = if let Some(rate) = &self.core.jmap.rate_authenticated { + self.core + .storage + .lookup + .is_rate_allowed( + format!("j:{}", access_token.primary_id).as_bytes(), + rate, + false, + ) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_none() + } else { + true + }; - if self - .lookup_store - .is_rate_allowed( - format!("j:{}", access_token.primary_id).as_bytes(), - &self.config.rate_authenticated, - false, - ) - .await - .map_err(|_| RequestError::internal_server_error())? - .is_none() - { + if is_rate_allowed { if let Some(in_flight_request) = limiter.concurrent_requests.is_allowed() { Ok(in_flight_request) } else if access_token.is_super_user() { @@ -84,21 +94,20 @@ impl JMAP { } pub async fn is_anonymous_allowed(&self, addr: &IpAddr) -> Result<(), RequestError> { - if self - .lookup_store - .is_rate_allowed( - format!("jreq:{}", addr).as_bytes(), - &self.config.rate_anonymous, - false, - ) - .await - .map_err(|_| RequestError::internal_server_error())? - .is_none() - { - Ok(()) - } else { - Err(RequestError::too_many_requests()) + if let Some(rate) = &self.core.jmap.rate_anonymous { + if self + .core + .storage + .lookup + .is_rate_allowed(format!("jreq:{}", addr).as_bytes(), rate, false) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_some() + { + return Err(RequestError::too_many_requests()); + } } + Ok(()) } pub fn is_upload_allowed(&self, access_token: &AccessToken) -> Result { @@ -116,39 +125,37 @@ impl JMAP { } pub async fn is_auth_allowed_soft(&self, addr: &IpAddr) -> Result<(), RequestError> { - if self - .lookup_store - .is_rate_allowed( - format!("jauth:{}", addr).as_bytes(), - &self.config.rate_authenticate_req, - true, - ) - .await - .map_err(|_| RequestError::internal_server_error())? - .is_none() - { - Ok(()) - } else { - Err(RequestError::too_many_auth_attempts()) + if let Some(rate) = &self.core.jmap.rate_authenticate_req { + if self + .core + .storage + .lookup + .is_rate_allowed(format!("jauth:{}", addr).as_bytes(), rate, true) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_some() + { + return Err(RequestError::too_many_auth_attempts()); + } } + Ok(()) } pub async fn is_auth_allowed_hard(&self, addr: &IpAddr) -> Result<(), RequestError> { - if self - .lookup_store - .is_rate_allowed( - format!("jauth:{}", addr).as_bytes(), - &self.config.rate_authenticate_req, - false, - ) - .await - .map_err(|_| RequestError::internal_server_error())? - .is_none() - { - Ok(()) - } else { - Err(RequestError::too_many_auth_attempts()) + if let Some(rate) = &self.core.jmap.rate_authenticate_req { + if self + .core + .storage + .lookup + .is_rate_allowed(format!("jauth:{}", addr).as_bytes(), rate, false) + .await + .map_err(|_| RequestError::internal_server_error())? + .is_some() + { + return Err(RequestError::too_many_auth_attempts()); + } } + Ok(()) } } diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index dc15974f..1d139195 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -55,7 +55,7 @@ impl JMAP { for blob_id in request.blob_ids { if self.has_access_blob(&blob_id, access_token).await? { let mut batch = BatchBuilder::new(); - let until = now() + self.config.upload_tmp_ttl; + let until = now() + self.core.jmap.upload_tmp_ttl; batch.with_account_id(account_id).set( BlobOp::Reserve { until, diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 0e054cc5..e2cc7e7d 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -48,7 +48,9 @@ impl JMAP { access_token: &AccessToken, ) -> Result>, MethodError> { if !self - .store + .core + .storage + .data .blob_has_access(&blob_id.hash, &blob_id.class) .await .map_err(|err| { @@ -129,7 +131,7 @@ impl JMAP { hash: &BlobHash, range: Range, ) -> Result>, MethodError> { - match self.blob_store.get_blob(hash.as_ref(), range).await { + match self.core.storage.blob.get_blob(hash.as_ref(), range).await { Ok(blob) => Ok(blob), Err(err) => { tracing::error!(event = "error", @@ -148,7 +150,9 @@ impl JMAP { access_token: &AccessToken, ) -> Result { Ok(self - .store + .core + .storage + .data .blob_has_access(&blob_id.hash, &blob_id.class) .await .map_err(|err| { diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index 7e0eaa94..5347147c 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -52,7 +52,7 @@ impl JMAP { access_token: &AccessToken, ) -> Result { let ids = request - .unwrap_blob_ids(self.config.get_max_objects)? + .unwrap_blob_ids(self.core.jmap.get_max_objects)? .unwrap_or_default(); let properties = request.unwrap_properties(&[ Property::Id, diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index 2c55d174..2645d89f 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -58,7 +58,7 @@ impl JMAP { }; let account_id = request.account_id.document_id(); - if request.create.len() > self.config.set_max_objects { + if request.create.len() > self.core.jmap.set_max_objects { return Err(MethodError::RequestTooLarge); } @@ -129,14 +129,14 @@ impl JMAP { DataSourceObject::Value(bytes) => bytes, }; - if bytes.len() + data.len() < self.config.upload_max_size { + if bytes.len() + data.len() < self.core.jmap.upload_max_size { data.extend(bytes); } else { response.not_created.append( create_id, SetError::too_large().with_description(format!( "Upload size exceeds maximum of {} bytes.", - self.config.upload_max_size + self.core.jmap.upload_max_size )), ); continue 'outer; @@ -153,26 +153,33 @@ impl JMAP { } // Enforce quota - let used = self.store.blob_quota(account_id).await.map_err(|err| { - tracing::error!(event = "error", + let used = self + .core + .storage + .data + .blob_quota(account_id) + .await + .map_err(|err| { + tracing::error!(event = "error", context = "blob_store", account_id = account_id, error = ?err, "Failed to obtain blob quota"); - MethodError::ServerPartialFail - })?; + MethodError::ServerPartialFail + })?; - if ((self.config.upload_tmp_quota_size > 0 - && used.bytes + data.len() > self.config.upload_tmp_quota_size) - || (self.config.upload_tmp_quota_amount > 0 - && used.count + 1 > self.config.upload_tmp_quota_amount)) + if ((self.core.jmap.upload_tmp_quota_size > 0 + && used.bytes + data.len() > self.core.jmap.upload_tmp_quota_size) + || (self.core.jmap.upload_tmp_quota_amount > 0 + && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) && !access_token.is_super_user() { response.not_created.append( create_id, SetError::over_quota().with_description(format!( "You have exceeded the blob upload quota of {} files or {} bytes.", - self.config.upload_tmp_quota_amount, self.config.upload_tmp_quota_size + self.core.jmap.upload_tmp_quota_amount, + self.core.jmap.upload_tmp_quota_size )), ); continue 'outer; @@ -212,7 +219,9 @@ impl JMAP { // Enforce quota let used = self - .store + .core + .storage + .data .blob_quota(account_id.document_id()) .await .map_err(|err| { @@ -224,15 +233,15 @@ impl JMAP { RequestError::internal_server_error() })?; - if ((self.config.upload_tmp_quota_size > 0 - && used.bytes + data.len() > self.config.upload_tmp_quota_size) - || (self.config.upload_tmp_quota_amount > 0 - && used.count + 1 > self.config.upload_tmp_quota_amount)) + if ((self.core.jmap.upload_tmp_quota_size > 0 + && used.bytes + data.len() > self.core.jmap.upload_tmp_quota_size) + || (self.core.jmap.upload_tmp_quota_amount > 0 + && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) && !access_token.is_super_user() { let err = Err(RequestError::over_blob_quota( - self.config.upload_tmp_quota_amount, - self.config.upload_tmp_quota_size, + self.core.jmap.upload_tmp_quota_amount, + self.core.jmap.upload_tmp_quota_size, )); #[cfg(feature = "test_mode")] @@ -265,7 +274,7 @@ impl JMAP { // First reserve the hash let hash = BlobHash::from(data); let mut batch = BatchBuilder::new(); - let until = now() + self.config.upload_tmp_ttl; + let until = now() + self.core.jmap.upload_tmp_ttl; batch.with_account_id(account_id).set( BlobOp::Reserve { @@ -276,16 +285,25 @@ impl JMAP { ); self.write_batch(batch).await?; - if !self.store.blob_exists(&hash).await.map_err(|err| { - tracing::error!( + if !self + .core + .storage + .data + .blob_exists(&hash) + .await + .map_err(|err| { + tracing::error!( event = "error", context = "put_blob", error = ?err, "Failed to verify blob hash existence."); - MethodError::ServerPartialFail - })? { + MethodError::ServerPartialFail + })? + { // Upload blob to store - self.blob_store + self.core + .storage + .blob .put_blob(hash.as_ref(), data) .await .map_err(|err| { diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index 4488716a..e54f7880 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -69,10 +69,10 @@ impl JMAP { } }; - let max_changes = if self.config.changes_max_results > 0 - && self.config.changes_max_results < request.max_changes.unwrap_or(0) + let max_changes = if self.core.jmap.changes_max_results > 0 + && self.core.jmap.changes_max_results < request.max_changes.unwrap_or(0) { - self.config.changes_max_results + self.core.jmap.changes_max_results } else { request.max_changes.unwrap_or(0) }; @@ -182,7 +182,9 @@ impl JMAP { collection: Collection, query: Query, ) -> Result { - self.store + self.core + .storage + .data .changes(account_id, collection, query) .await .map_err(|err| { diff --git a/crates/jmap/src/changes/state.rs b/crates/jmap/src/changes/state.rs index 08167e94..9196a1b2 100644 --- a/crates/jmap/src/changes/state.rs +++ b/crates/jmap/src/changes/state.rs @@ -35,7 +35,13 @@ impl JMAP { collection: impl Into, ) -> Result { let collection = collection.into(); - match self.store.get_last_change_id(account_id, collection).await { + match self + .core + .storage + .data + .get_last_change_id(account_id, collection) + .await + { Ok(id) => Ok(id.into()), Err(err) => { tracing::error!(event = "error", diff --git a/crates/jmap/src/changes/write.rs b/crates/jmap/src/changes/write.rs index d2baa0c7..397d2c0c 100644 --- a/crates/jmap/src/changes/write.rs +++ b/crates/jmap/src/changes/write.rs @@ -35,7 +35,7 @@ impl JMAP { pub async fn assign_change_id(&self, _: u32) -> Result { self.generate_snowflake_id() - /*self.store + /*self.core.storage.data .assign_change_id(account_id) .await .map_err(|err| { @@ -49,7 +49,7 @@ impl JMAP { } pub fn generate_snowflake_id(&self) -> Result { - self.snowflake_id.generate().ok_or_else(|| { + self.inner.snowflake_id.generate().ok_or_else(|| { tracing::error!( event = "error", context = "change_log", @@ -71,14 +71,19 @@ impl JMAP { let mut builder = BatchBuilder::new(); builder.with_account_id(account_id).custom(changes); - self.store.write(builder.build()).await.map_err(|err| { - tracing::error!( + self.core + .storage + .data + .write(builder.build()) + .await + .map_err(|err| { + tracing::error!( event = "error", context = "change_log", error = ?err, "Failed to write changes."); - MethodError::ServerPartialFail - })?; + MethodError::ServerPartialFail + })?; Ok(state) } diff --git a/crates/jmap/src/email/cache.rs b/crates/jmap/src/email/cache.rs index 89e60e51..4ab6f538 100644 --- a/crates/jmap/src/email/cache.rs +++ b/crates/jmap/src/email/cache.rs @@ -46,7 +46,9 @@ impl JMAP { ) -> Result, MethodError> { // Obtain current state let modseq = self - .store + .core + .storage + .data .get_last_change_id(account_id, Collection::Thread) .map_err(|err| { tracing::error!(event = "error", @@ -60,7 +62,7 @@ impl JMAP { // Lock the cache let thread_cache = if let Some(thread_cache) = - self.cache_threads.get(&account_id).and_then(|t| { + self.inner.cache_threads.get(&account_id).and_then(|t| { if t.modseq.unwrap_or(0) >= modseq.unwrap_or(0) { Some(t) } else { @@ -82,7 +84,9 @@ impl JMAP { .collect(), modseq, }); - self.cache_threads.insert(account_id, thread_cache.clone()); + self.inner + .cache_threads + .insert(account_id, thread_cache.clone()); thread_cache }; diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index ae9e7d46..7b9f7a88 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -435,17 +435,22 @@ impl JMAP { .custom(EmailIndexBuilder::set(metadata)) .custom(changes); - self.store.write(batch.build()).await.map_err(|err| { - tracing::error!( + self.core + .storage + .data + .write(batch.build()) + .await + .map_err(|err| { + tracing::error!( event = "error", context = "email_copy", error = ?err, "Failed to write message to database."); - MethodError::ServerPartialFail - })?; + MethodError::ServerPartialFail + })?; // Request FTS index - let _ = self.housekeeper_tx.send(Event::IndexStart).await; + let _ = self.inner.housekeeper_tx.send(Event::IndexStart).await; Ok(Ok(email)) } diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index a3ab9206..b6606515 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -29,6 +29,7 @@ use crate::{ JMAP, }; use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit}; +use common::AuthResult; use jmap_proto::types::{collection::Collection, property::Property}; use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary}; use mail_parser::{decoders::base64::base64_decode, Message, MessageParser, MimeHeaders}; diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index ad32e585..d43e2cb5 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -53,7 +53,7 @@ impl JMAP { mut request: GetRequest, access_token: &AccessToken, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ Property::Id, Property::BlobId, @@ -108,7 +108,7 @@ impl JMAP { } else { let document_ids = message_ids .iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .collect::>(); self.get_cached_thread_ids(account_id, document_ids.iter().copied()) .await diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index cff0279b..08857eba 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -141,7 +141,7 @@ impl JMAP { keywords: email.keywords, received_at: email.received_at.map(|r| r.into()), skip_duplicates: true, - encrypt: self.config.encrypt && self.config.encrypt_append, + encrypt: self.core.jmap.encrypt && self.core.jmap.encrypt_append, }) .await { diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 6c0b5449..611b3706 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -108,7 +108,7 @@ impl JMAP { })?; // Check for Spam headers - if let Some((header_name, header_value)) = &self.config.spam_header { + if let Some((header_name, header_value)) = &self.core.jmap.spam_header { if params.mailbox_ids == [INBOX_ID] && message.root_part().headers().iter().any(|header| { &header.name == header_name @@ -164,7 +164,9 @@ impl JMAP { if params.skip_duplicates && !message_id.is_empty() && !self - .store + .core + .storage + .data .filter( params.account_id, Collection::Email, @@ -261,7 +263,9 @@ impl JMAP { // Obtain a documentId and changeId let document_id = self - .store + .core + .storage + .data .assign_document_id(params.account_id, Collection::Email) .await .map_err(|err| { @@ -327,7 +331,9 @@ impl JMAP { thread_id } else { let thread_id = self - .store + .core + .storage + .data .assign_document_id(params.account_id, Collection::Thread) .await .map_err(|err| { @@ -371,17 +377,22 @@ impl JMAP { ), blob_id.hash.clone(), ); - self.store.write(batch.build()).await.map_err(|err| { - tracing::error!( + self.core + .storage + .data + .write(batch.build()) + .await + .map_err(|err| { + tracing::error!( event = "error", context = "email_ingest", error = ?err, "Failed to write message to database."); - IngestError::Temporary - })?; + IngestError::Temporary + })?; // Request FTS index - let _ = self.housekeeper_tx.send(Event::IndexStart).await; + let _ = self.inner.housekeeper_tx.send(Event::IndexStart).await; tracing::debug!( context = "email_ingest", @@ -436,7 +447,9 @@ impl JMAP { } filters.push(Filter::End); let results = self - .store + .core + .storage + .data .filter(account_id, Collection::Email, filters) .await .map_err(|err| { @@ -522,7 +535,9 @@ impl JMAP { { if thread_id != old_thread_id { for document_id in self - .store + .core + .storage + .data .get_bitmap(BitmapKey { account_id, collection: Collection::Email.into(), @@ -558,7 +573,7 @@ impl JMAP { } batch.custom(changes); - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => return Ok(Some(thread_id)), Err(store::Error::AssertValueFailed) if try_count < MAX_RETRIES => { let backoff = rand::thread_rng().gen_range(50..=300); @@ -585,7 +600,9 @@ impl JMAP { .with_collection(Collection::Mailbox) .update_document(mailbox_id) .add_and_get(Property::EmailIds, 1); - self.store + self.core + .storage + .data .write(batch.build()) .await .map(|v| v.expect("UID next") as u32) diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index fd8920ea..9ec71243 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -46,7 +46,7 @@ impl JMAP { request: ParseEmailRequest, access_token: &AccessToken, ) -> Result { - if request.blob_ids.len() > self.config.mail_parse_max_items { + if request.blob_ids.len() > self.core.jmap.mail_parse_max_items { return Err(MethodError::RequestTooLarge); } let properties = request.properties.unwrap_or_else(|| { diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 83eaa383..6ba0e625 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -79,17 +79,17 @@ impl JMAP { fts_filters.push(FtsFilter::has_text_detect( Field::Header(HeaderName::Subject), &text, - self.config.default_language, + self.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( Field::Body, &text, - self.config.default_language, + self.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( Field::Attachment, text, - self.config.default_language, + self.core.jmap.default_language, )); fts_filters.push(FtsFilter::End); } @@ -116,12 +116,12 @@ impl JMAP { Filter::Subject(text) => fts_filters.push(FtsFilter::has_text_detect( Field::Header(HeaderName::Subject), text, - self.config.default_language, + self.core.jmap.default_language, )), Filter::Body(text) => fts_filters.push(FtsFilter::has_text_detect( Field::Body, text, - self.config.default_language, + self.core.jmap.default_language, )), Filter::Header(header) => { let mut header = header.into_iter(); diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index b84cf7bb..20a793af 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -599,8 +599,9 @@ impl JMAP { // Check attachment sizes if !is_multipart { size_attachments += parts.last().unwrap().size(); - if self.config.mail_attachments_max_size > 0 - && size_attachments > self.config.mail_attachments_max_size + if self.core.jmap.mail_attachments_max_size > 0 + && size_attachments + > self.core.jmap.mail_attachments_max_size { response.not_created.append( id, @@ -608,7 +609,7 @@ impl JMAP { .with_property(property) .with_description(format!( "Message exceeds maximum size of {} bytes.", - self.config.mail_attachments_max_size + self.core.jmap.mail_attachments_max_size )), ); continue 'create; @@ -741,7 +742,7 @@ impl JMAP { keywords, received_at, skip_duplicates: false, - encrypt: self.config.encrypt && self.config.encrypt_append, + encrypt: self.core.jmap.encrypt && self.core.jmap.encrypt_append, }) .await { @@ -986,7 +987,7 @@ impl JMAP { // Write changes if !batch.is_empty() { - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => { // Add to updated list response.updated.append(id, None); @@ -1240,7 +1241,7 @@ impl JMAP { } // Commit batch - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => (), Err(store::Error::AssertValueFailed) => { return Ok(Err(SetError::forbidden().with_description( @@ -1258,7 +1259,7 @@ impl JMAP { } // Request FTS index - let _ = self.housekeeper_tx.send(Event::IndexStart).await; + let _ = self.inner.housekeeper_tx.send(Event::IndexStart).await; Ok(Ok(changes)) } diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 2c7b59d2..5f2c1133 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -47,14 +47,14 @@ impl JMAP { let mut include_term = true; let mut terms = vec![]; let mut is_exact = false; - let mut language = self.config.default_language; + let mut language = self.core.jmap.default_language; for cond in request.filter { match cond { Filter::Text(text) | Filter::Subject(text) | Filter::Body(text) => { if include_term { let (text, language_) = - Language::detect(text, self.config.default_language); + Language::detect(text, self.core.jmap.default_language); language = language_; if (text.starts_with('"') && text.ends_with('"')) || (text.starts_with('\'') && text.ends_with('\'')) @@ -99,7 +99,7 @@ impl JMAP { not_found: vec![], }; - if email_ids.len() > self.config.snippet_max_results { + if email_ids.len() > self.core.jmap.snippet_max_results { return Err(MethodError::RequestTooLarge); } diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 1e02c72a..2181b0d1 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -42,7 +42,7 @@ impl JMAP { &self, mut request: GetRequest, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ Property::Id, Property::Name, @@ -60,7 +60,7 @@ impl JMAP { } else { identity_ids .iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; @@ -129,6 +129,8 @@ impl JMAP { // Obtain principal let principal = self + .core + .storage .directory .query(QueryBy::Id(account_id), false) .await @@ -181,14 +183,19 @@ impl JMAP { ); identity_ids.insert(identity_id); } - self.store.write(batch.build()).await.map_err(|err| { - tracing::error!( + self.core + .storage + .data + .write(batch.build()) + .await + .map_err(|err| { + tracing::error!( event = "error", context = "identity_get_or_create", error = ?err, "Failed to create identities."); - MethodError::ServerPartialFail - })?; + MethodError::ServerPartialFail + })?; Ok(identity_ids) } diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index d12c553c..8f8de881 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -47,7 +47,7 @@ impl JMAP { .get_document_ids(account_id, Collection::Identity) .await? .unwrap_or_default(); - let mut response = SetResponse::from_request(&request, self.config.set_max_objects)?; + let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy(); // Process creates @@ -74,6 +74,8 @@ impl JMAP { // Validate email address if let Value::Text(email) = identity.get(&Property::Email) { if !self + .core + .storage .directory .query(QueryBy::Id(account_id), false) .await diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index aea61960..9f7e194f 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -23,11 +23,10 @@ use std::{collections::hash_map::RandomState, fmt::Display, sync::Arc, time::Duration}; -use ::sieve::{Compiler, Runtime}; -use api::session::BaseCapabilities; use auth::{oauth::OAuthCode, rate_limit::ConcurrencyLimiters, AccessToken}; +use common::{Core, DeliveryEvent, SharedCore}; use dashmap::DashMap; -use directory::{Directories, Directory, QueryBy}; +use directory::QueryBy; use email::cache::Threads; use jmap_proto::{ error::method::MethodError, @@ -37,13 +36,12 @@ use jmap_proto::{ }, types::{collection::Collection, property::Property}, }; -use mail_parser::HeaderName; -use nlp::language::Language; use services::{ delivery::spawn_delivery_manager, housekeeper::{self, init_housekeeper, spawn_housekeeper}, state::{self, init_state_manager, spawn_state_manager}, }; + use smtp::core::SMTP; use store::{ fts::FtsFilter, @@ -52,17 +50,14 @@ use store::{ write::{ key::DeserializeBigEndian, BatchBuilder, BitmapClass, DirectoryClass, TagValue, ValueClass, }, - BitmapKey, BlobStore, Deserialize, FtsStore, IterateParams, LookupStore, Store, Stores, - ValueKey, U32_LEN, + BitmapKey, Deserialize, IterateParams, ValueKey, U32_LEN, }; use tokio::sync::mpsc; use utils::{ - config::{Rate, Servers}, - ipc::DeliveryEvent, + config::Config, lru_cache::{LruCache, LruCached}, map::ttl_dashmap::{TtlDashMap, TtlMap}, snowflake::SnowflakeIdGenerator, - UnwrapFailure, }; pub mod api; @@ -84,14 +79,21 @@ pub mod websocket; pub const LONG_SLUMBER: Duration = Duration::from_secs(60 * 60 * 24); +#[derive(Clone)] pub struct JMAP { - pub store: Store, - pub blob_store: BlobStore, - pub fts_store: FtsStore, - pub lookup_store: LookupStore, - pub config: Config, - pub directory: Arc, + pub core: Arc, + pub inner: Arc, + pub smtp: SMTP, +} +#[derive(Clone)] +pub struct JmapInstance { + pub core: SharedCore, + pub jmap_inner: Arc, + pub smtp_inner: Arc, +} + +pub struct Inner { pub sessions: TtlDashMap, pub access_tokens: TtlDashMap>, pub snowflake_id: SnowflakeIdGenerator, @@ -101,74 +103,8 @@ pub struct JMAP { pub state_tx: mpsc::Sender, pub housekeeper_tx: mpsc::Sender, - pub smtp: Arc, pub cache_threads: LruCache>, - - pub sieve_compiler: Compiler, - pub sieve_runtime: Runtime<()>, -} - -pub struct Config { - pub default_language: Language, - pub query_max_results: usize, - pub changes_max_results: usize, - pub snippet_max_results: usize, - - pub request_max_size: usize, - pub request_max_calls: usize, - pub request_max_concurrent: u64, - - pub get_max_objects: usize, - pub set_max_objects: usize, - - pub upload_max_size: usize, - pub upload_max_concurrent: u64, - - pub upload_tmp_quota_size: usize, - pub upload_tmp_quota_amount: usize, - pub upload_tmp_ttl: u64, - - pub mailbox_max_depth: usize, - pub mailbox_name_max_len: usize, - pub mail_attachments_max_size: usize, - pub mail_parse_max_items: usize, - pub mail_max_size: usize, - - pub sieve_max_script_name: usize, - pub sieve_max_scripts: usize, - - pub session_cache_ttl: Duration, - pub rate_authenticated: Rate, - pub rate_authenticate_req: Rate, - pub rate_anonymous: Rate, - pub rate_use_forwarded: bool, - - pub event_source_throttle: Duration, - pub push_max_total: usize, - - pub web_socket_throttle: Duration, - pub web_socket_timeout: Duration, - pub web_socket_heartbeat: Duration, - - pub oauth_key: String, - pub oauth_expiry_user_code: u64, - pub oauth_expiry_auth_code: u64, - pub oauth_expiry_token: u64, - pub oauth_expiry_refresh_token: u64, - pub oauth_expiry_refresh_token_renew: u64, - pub oauth_max_auth_attempts: u32, - - pub spam_header: Option<(HeaderName<'static>, String)>, - - pub http_headers: Vec<(hyper::header::HeaderName, hyper::header::HeaderValue)>, - - pub encrypt: bool, - pub encrypt_append: bool, - - pub principal_allow_lookups: bool, - - pub capabilities: BaseCapabilities, } #[derive(Debug)] @@ -180,213 +116,56 @@ pub enum IngestError { impl JMAP { pub async fn init( - config: &utils::config::Config, - stores: &Stores, - directories: &Directories, - servers: &mut Servers, + config: &mut Config, delivery_rx: mpsc::Receiver, - smtp: Arc, - ) -> Result, String> { + core: SharedCore, + smtp_inner: Arc, + ) -> JmapInstance { // Init state manager and housekeeper let (state_tx, state_rx) = init_state_manager(); let (housekeeper_tx, housekeeper_rx) = init_housekeeper(); let shard_amount = config - .property::("cache.shard")? + .property_::("cache.shard") .unwrap_or(32) .next_power_of_two() as usize; - let capacity = config.property("cache.capacity")?.unwrap_or(100); + let capacity = config.property_("cache.capacity").unwrap_or(100); - let jmap_server = Arc::new(JMAP { - directory: directories - .directories - .get(config.value_require("storage.directory")?) - .failed(&format!( - "Unable to find directory '{}'", - config.value_require("storage.directory")? - )) - .clone(), - snowflake_id: config - .property::("storage.cluster.node-id")? - .map(SnowflakeIdGenerator::with_node_id) - .unwrap_or_else(SnowflakeIdGenerator::new), - store: stores.get_store(config, "storage.data")?, - fts_store: stores.get_fts_store(config, "storage.fts")?, - blob_store: stores.get_blob_store(config, "storage.blob")?, - lookup_store: stores.get_lookup_store(config, "storage.lookup")?, - config: Config::new(config).failed("Invalid configuration file"), + let inner = Inner { sessions: TtlDashMap::with_capacity(capacity, shard_amount), access_tokens: TtlDashMap::with_capacity(capacity, shard_amount), + snowflake_id: config + .property_::("cluster.node-id") + .map(SnowflakeIdGenerator::with_node_id) + .unwrap_or_default(), concurrency_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( capacity, RandomState::default(), shard_amount, ), oauth_codes: TtlDashMap::with_capacity(capacity, shard_amount), - cache_threads: LruCache::with_capacity( - config.property("cache.thread.size")?.unwrap_or(2048), - ), state_tx, housekeeper_tx, - smtp, - sieve_compiler: Compiler::new() - .with_max_script_size( - config - .property("sieve.untrusted.limits.script-size")? - .unwrap_or(1024 * 1024), - ) - .with_max_string_size( - config - .property("sieve.untrusted.limits.string-length")? - .unwrap_or(4096), - ) - .with_max_variable_name_size( - config - .property("sieve.untrusted.limits.variable-name-length")? - .unwrap_or(32), - ) - .with_max_nested_blocks( - config - .property("sieve.untrusted.limits.nested-blocks")? - .unwrap_or(15), - ) - .with_max_nested_tests( - config - .property("sieve.untrusted.limits.nested-tests")? - .unwrap_or(15), - ) - .with_max_nested_foreverypart( - config - .property("sieve.untrusted.limits.nested-foreverypart")? - .unwrap_or(3), - ) - .with_max_match_variables( - config - .property("sieve.untrusted.limits.match-variables")? - .unwrap_or(30), - ) - .with_max_local_variables( - config - .property("sieve.untrusted.limits.local-variables")? - .unwrap_or(128), - ) - .with_max_header_size( - config - .property("sieve.untrusted.limits.header-size")? - .unwrap_or(1024), - ) - .with_max_includes( - config - .property("sieve.untrusted.limits.includes")? - .unwrap_or(3), - ), - sieve_runtime: Runtime::new() - .with_max_nested_includes( - config - .property("sieve.untrusted.limits.nested-includes")? - .unwrap_or(3), - ) - .with_cpu_limit( - config - .property("sieve.untrusted.limits.cpu")? - .unwrap_or(5000), - ) - .with_max_variable_size( - config - .property("sieve.untrusted.limits.variable-size")? - .unwrap_or(4096), - ) - .with_max_redirects( - config - .property("sieve.untrusted.limits.redirects")? - .unwrap_or(1), - ) - .with_max_received_headers( - config - .property("sieve.untrusted.limits.received-headers")? - .unwrap_or(10), - ) - .with_max_header_size( - config - .property("sieve.untrusted.limits.header-size")? - .unwrap_or(1024), - ) - .with_max_out_messages( - config - .property("sieve.untrusted.limits.outgoing-messages")? - .unwrap_or(3), - ) - .with_default_vacation_expiry( - config - .property::("sieve.untrusted.default-expiry.vacation")? - .unwrap_or(Duration::from_secs(30 * 86400)) - .as_secs(), - ) - .with_default_duplicate_expiry( - config - .property::("sieve.untrusted.default-expiry.duplicate")? - .unwrap_or(Duration::from_secs(7 * 86400)) - .as_secs(), - ) - .without_capabilities( - config - .values("sieve.untrusted.disable-capabilities") - .map(|(_, v)| v), - ) - .with_valid_notification_uris({ - let values = config - .values("sieve.untrusted.notification-uris") - .map(|(_, v)| v.to_string()) - .collect::>(); - if !values.is_empty() { - values - } else { - vec!["mailto".to_string()] - } - }) - .with_protected_headers({ - let values = config - .values("sieve.untrusted.protected-headers") - .map(|(_, v)| v.to_string()) - .collect::>(); - if !values.is_empty() { - values - } else { - vec![ - "Original-Subject".to_string(), - "Original-From".to_string(), - "Received".to_string(), - "Auto-Submitted".to_string(), - ] - } - }) - .with_vacation_default_subject( - config - .value("sieve.untrusted.vacation.default-subject") - .unwrap_or("Automated reply") - .to_string(), - ) - .with_vacation_subject_prefix( - config - .value("sieve.untrusted.vacation.subject-prefix") - .unwrap_or("Auto: ") - .to_string(), - ) - .with_env_variable("name", "Stalwart JMAP") - .with_env_variable("version", env!("CARGO_PKG_VERSION")) - .with_env_variable("location", "MS") - .with_env_variable("phase", "during"), - }); + cache_threads: LruCache::with_capacity( + config.property_("cache.thread.size").unwrap_or(2048), + ), + }; + + let jmap_instance = JmapInstance { + core, + jmap_inner: Arc::new(inner), + smtp_inner, + }; // Spawn delivery manager - spawn_delivery_manager(jmap_server.clone(), delivery_rx); + spawn_delivery_manager(jmap_instance.clone(), delivery_rx); // Spawn state manager - spawn_state_manager(jmap_server.clone(), config, state_rx); + spawn_state_manager(jmap_instance.clone(), state_rx); // Spawn housekeeper - spawn_housekeeper(jmap_server.clone(), config, servers, housekeeper_rx); + spawn_housekeeper(jmap_instance.clone(), housekeeper_rx); - Ok(jmap_server) + jmap_instance } pub async fn assign_document_id( @@ -394,7 +173,9 @@ impl JMAP { account_id: u32, collection: Collection, ) -> Result { - self.store + self.core + .storage + .data .assign_document_id(account_id, collection) .await .map_err(|err| { @@ -419,7 +200,9 @@ impl JMAP { { let property = property.as_ref(); match self - .store + .core + .storage + .data .get_value::(ValueKey { account_id, collection: collection.into(), @@ -460,7 +243,9 @@ impl JMAP { let expected_results = iterate.len(); let mut results = Vec::with_capacity(expected_results); - self.store + self.core + .storage + .data .iterate( IterateParams::new( ValueKey { @@ -507,7 +292,9 @@ impl JMAP { collection: Collection, ) -> Result, MethodError> { match self - .store + .core + .storage + .data .get_bitmap(BitmapKey::document_ids(account_id, collection)) .await { @@ -533,7 +320,9 @@ impl JMAP { ) -> Result, MethodError> { let property = property.as_ref(); match self - .store + .core + .storage + .data .get_bitmap(BitmapKey { account_id, collection: collection.into(), @@ -565,7 +354,7 @@ impl JMAP { collection: Collection, ) -> Result { Ok( - SetResponse::from_request(request, self.config.set_max_objects)?.with_state( + SetResponse::from_request(request, self.core.jmap.set_max_objects)?.with_state( self.assert_state( request.account_id.document_id(), collection, @@ -584,7 +373,9 @@ impl JMAP { Ok(if access_token.primary_id == account_id { access_token.quota as i64 } else { - self.directory + self.core + .storage + .directory .query(QueryBy::Id(account_id), false) .await .map_err(|err| { @@ -602,7 +393,9 @@ impl JMAP { } pub async fn get_used_quota(&self, account_id: u32) -> Result { - self.store + self.core + .storage + .data .get_counter(DirectoryClass::UsedQuota(account_id)) .await .map_err(|err| { @@ -622,7 +415,9 @@ impl JMAP { collection: Collection, filters: Vec, ) -> Result { - self.store + self.core + .storage + .data .filter(account_id, collection, filters) .await .map_err(|err| { @@ -643,7 +438,9 @@ impl JMAP { collection: Collection, filters: Vec>, ) -> Result { - self.fts_store + self.core + .storage + .fts .query(account_id, collection, filters) .await .map_err(|err| { @@ -666,15 +463,15 @@ impl JMAP { let total = result_set.results.len() as usize; let (limit_total, limit) = if let Some(limit) = request.limit { if limit > 0 { - let limit = std::cmp::min(limit, self.config.query_max_results); + let limit = std::cmp::min(limit, self.core.jmap.query_max_results); (std::cmp::min(limit, total), limit) } else { (0, 0) } } else { ( - std::cmp::min(self.config.query_max_results, total), - self.config.query_max_results, + std::cmp::min(self.core.jmap.query_max_results, total), + self.core.jmap.query_max_results, ) }; Ok(( @@ -718,7 +515,13 @@ impl JMAP { let collection = result_set.collection; let account_id = result_set.account_id; response.update_results( - match self.store.sort(result_set, comparators, paginate).await { + match self + .core + .storage + .data + .sort(result_set, comparators, paginate) + .await + { Ok(result) => result, Err(err) => { tracing::error!(event = "error", @@ -736,7 +539,9 @@ impl JMAP { } pub async fn write_batch(&self, batch: BatchBuilder) -> Result<(), MethodError> { - self.store + self.core + .storage + .data .write(batch.build()) .await .map(|_| ()) @@ -764,6 +569,20 @@ impl JMAP { } } +impl From for JMAP { + fn from(value: JmapInstance) -> Self { + let core = value.core.load().clone(); + JMAP { + smtp: SMTP { + core: core.clone(), + inner: value.smtp_inner, + }, + core, + inner: value.jmap_inner, + } + } +} + trait UpdateResults: Sized { fn update_results(&mut self, sorted_results: SortedResultSet) -> Result<(), MethodError>; } diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index f143e6fa..40529e1b 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -40,7 +40,7 @@ impl JMAP { mut request: GetRequest, access_token: &AccessToken, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ Property::Id, Property::Name, @@ -67,7 +67,7 @@ impl JMAP { } else { mailbox_ids .iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; @@ -338,7 +338,7 @@ impl JMAP { } }) .collect::>(); - if path.is_empty() || path.len() > self.config.mailbox_max_depth { + if path.is_empty() || path.len() > self.core.jmap.mailbox_max_depth { return Ok(None); } diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index b7ebd221..5f5d9277 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -146,7 +146,7 @@ impl JMAP { let mut keep = false; let mut jmap_id = document_id + 1; - for _ in 0..self.config.mailbox_max_depth { + for _ in 0..self.core.jmap.mailbox_max_depth { if let Some(&parent_id) = hierarchy.get(&jmap_id) { if parent_id == 0 { keep = true; @@ -213,7 +213,7 @@ impl JMAP { let mut stack = Vec::new(); let mut jmap_id = 0; - 'outer: for _ in 0..(response.ids.len() * 10 * self.config.mailbox_max_depth) { + 'outer: for _ in 0..(response.ids.len() * 10 * self.core.jmap.mailbox_max_depth) { let (mut children, mut it) = if let Some(children) = tree.remove(&jmap_id) { (children, response.ids.iter()) } else if let Some(prev) = stack.pop() { diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 0a3ed960..50373c59 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -136,7 +136,7 @@ impl JMAP { batch.create_document(document_id).custom(builder); changes.log_insert(Collection::Mailbox, document_id); ctx.mailbox_ids.insert(document_id); - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => { ctx.response.created(id, document_id); } @@ -235,7 +235,7 @@ impl JMAP { batch.update_document(document_id).custom(builder); if !batch.is_empty() { - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => { changes.log_update(Collection::Mailbox, document_id); } @@ -403,7 +403,7 @@ impl JMAP { .assert_value(Property::MailboxIds, &mailbox_ids) .value(Property::MailboxIds, mailbox_ids.inner, F_VALUE) .value(Property::MailboxIds, document_id, F_BITMAP | F_CLEAR); - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => changes.log_update( Collection::Email, Id::from_parts(thread_id, message_id), @@ -497,7 +497,7 @@ impl JMAP { .value(Property::EmailIds, (), F_VALUE | F_CLEAR) .custom(ObjectIndexBuilder::new(SCHEMA).with_current(mailbox)); - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => { changes.log_delete(Collection::Mailbox, document_id); Ok(Ok(did_remove_emails)) @@ -542,7 +542,7 @@ impl JMAP { let value = match (&property, value) { (Property::Name, MaybePatchValue::Value(Value::Text(value))) => { let value = value.trim(); - if !value.is_empty() && value.len() < self.config.mailbox_name_max_len { + if !value.is_empty() && value.len() < self.core.jmap.mailbox_name_max_len { Value::Text(value.to_string()) } else { return Ok(Err(SetError::invalid_properties() @@ -633,7 +633,7 @@ impl JMAP { .map_or(u32::MAX, |(mailbox_id, _)| *mailbox_id + 1); let mut mailbox_parent_id = mailbox_parent_id.document_id(); let mut success = false; - for depth in 0..self.config.mailbox_max_depth { + for depth in 0..self.core.jmap.mailbox_max_depth { if mailbox_parent_id == current_mailbox_id { return Ok(Err(SetError::invalid_properties() .with_property(Property::ParentId) @@ -857,14 +857,19 @@ impl JMAP { ); mailbox_ids.insert(mailbox_id); } - self.store.write(batch.build()).await.map_err(|err| { - tracing::error!( + self.core + .storage + .data + .write(batch.build()) + .await + .map_err(|err| { + tracing::error!( event = "error", context = "mailbox_get_or_create", error = ?err, "Failed to create mailboxes."); - MethodError::ServerPartialFail - })?; + MethodError::ServerPartialFail + })?; Ok(mailbox_ids) } @@ -903,7 +908,7 @@ impl JMAP { .with_collection(Collection::Mailbox); for name in path { - if name.len() > self.config.mailbox_name_max_len { + if name.len() > self.core.jmap.mailbox_name_max_len { return Ok(None); } diff --git a/crates/jmap/src/principal/get.rs b/crates/jmap/src/principal/get.rs index 4e8dfbc0..8b12d1d5 100644 --- a/crates/jmap/src/principal/get.rs +++ b/crates/jmap/src/principal/get.rs @@ -36,7 +36,7 @@ impl JMAP { &self, mut request: GetRequest, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ Property::Id, Property::Type, @@ -55,7 +55,7 @@ impl JMAP { } else { email_submission_ids .iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; @@ -69,6 +69,8 @@ impl JMAP { for id in ids { // Obtain the principal let principal = if let Some(principal) = self + .core + .storage .directory .query(QueryBy::Id(id.document_id()), false) .await diff --git a/crates/jmap/src/principal/query.rs b/crates/jmap/src/principal/query.rs index 1946bbcb..f42608fb 100644 --- a/crates/jmap/src/principal/query.rs +++ b/crates/jmap/src/principal/query.rs @@ -48,6 +48,8 @@ impl JMAP { match cond { Filter::Name(name) => { if let Some(principal) = self + .core + .storage .directory .query(QueryBy::Name(name.as_str()), false) .await @@ -67,8 +69,8 @@ impl JMAP { Filter::Email(email) => { let mut ids = RoaringBitmap::new(); for id in self - .directory - .email_to_ids(&email) + .core + .email_to_ids(&self.core.storage.directory, &email) .await .map_err(|_| MethodError::ServerPartialFail)? { diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index ca228208..4fc9a00f 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -44,7 +44,7 @@ impl JMAP { mut request: GetRequest, access_token: &AccessToken, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ Property::Id, Property::DeviceClientId, @@ -62,7 +62,7 @@ impl JMAP { } else { push_ids .iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; @@ -119,7 +119,9 @@ impl JMAP { pub async fn fetch_push_subscriptions(&self, account_id: u32) -> store::Result { let mut subscriptions = Vec::new(); let document_ids = self - .store + .core + .storage + .data .get_bitmap(BitmapKey::document_ids( account_id, Collection::PushSubscription, @@ -131,7 +133,9 @@ impl JMAP { for document_id in document_ids { let mut subscription = self - .store + .core + .storage + .data .get_value::>(ValueKey { account_id, collection: Collection::PushSubscription.into(), diff --git a/crates/jmap/src/push/manager.rs b/crates/jmap/src/push/manager.rs index 283467cf..bfee8259 100644 --- a/crates/jmap/src/push/manager.rs +++ b/crates/jmap/src/push/manager.rs @@ -25,9 +25,8 @@ use base64::{engine::general_purpose, Engine}; use jmap_proto::types::id::Id; use store::ahash::{AHashMap, AHashSet}; use tokio::sync::mpsc; -use utils::{config::Config, UnwrapFailure}; -use crate::{api::StateChangeResponse, services::IPC_CHANNEL_BUFFER, LONG_SLUMBER}; +use crate::{api::StateChangeResponse, services::IPC_CHANNEL_BUFFER, JmapInstance, LONG_SLUMBER}; use super::{ece::ece_encrypt, EncryptionKeys, Event, PushServer, PushUpdate}; @@ -37,29 +36,10 @@ use std::{ time::{Duration, Instant}, }; -pub fn spawn_push_manager(settings: &Config) -> mpsc::Sender { +pub fn spawn_push_manager(core: JmapInstance) -> mpsc::Sender { let (push_tx_, mut push_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); let push_tx = push_tx_.clone(); - let push_attempt_interval: Duration = settings - .property_or_default("jmap.push.attempts.interval", "1m") - .failed("Invalid configuration"); - let push_attempts_max: u32 = settings - .property_or_default("jmap.push.attempts.max", "3") - .failed("Invalid configuration"); - let push_retry_interval: Duration = settings - .property_or_default("jmap.push.retry.interval", "1s") - .failed("Invalid configuration"); - let push_timeout: Duration = settings - .property_or_default("jmap.push.timeout.request", "10s") - .failed("Invalid configuration"); - let push_verify_timeout: Duration = settings - .property_or_default("jmap.push.timeout.verify", "1m") - .failed("Invalid configuration"); - let push_throttle: Duration = settings - .property_or_default("jmap.push.throttle", "1s") - .failed("Invalid configuration"); - tokio::spawn(async move { let mut subscriptions = AHashMap::default(); let mut last_verify: AHashMap = AHashMap::default(); @@ -68,6 +48,15 @@ pub fn spawn_push_manager(settings: &Config) -> mpsc::Sender { let mut retry_ids = AHashSet::default(); loop { + // Load settings + let core_ = core.core.load(); + let push_attempt_interval = core_.jmap.push_attempt_interval; + let push_attempts_max = core_.jmap.push_attempts_max; + let push_retry_interval = core_.jmap.push_retry_interval; + let push_timeout = core_.jmap.push_timeout; + let push_verify_timeout = core_.jmap.push_verify_timeout; + let push_throttle = core_.jmap.push_throttle; + match tokio::time::timeout(retry_timeout, push_rx.recv()).await { Ok(Some(event)) => match event { Event::Update { updates } => { diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 0de17765..42d92206 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -56,14 +56,14 @@ impl JMAP { .get_document_ids(account_id, Collection::PushSubscription) .await? .unwrap_or_default(); - let mut response = SetResponse::from_request(&request, self.config.set_max_objects)?; + let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy(); // Process creates 'create: for (id, object) in request.unwrap_create() { let mut push = Object::with_capacity(object.properties.len()); - if push_ids.len() as usize >= self.config.push_max_total { + if push_ids.len() as usize >= self.core.jmap.push_max_total { response.not_created.append(id, SetError::forbidden().with_description( "There are too many subscriptions, please delete some before adding a new one.", )); diff --git a/crates/jmap/src/quota/get.rs b/crates/jmap/src/quota/get.rs index d6cda5df..93045827 100644 --- a/crates/jmap/src/quota/get.rs +++ b/crates/jmap/src/quota/get.rs @@ -36,7 +36,7 @@ impl JMAP { mut request: GetRequest, access_token: &AccessToken, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ Property::Id, Property::ResourceType, diff --git a/crates/jmap/src/services/delivery.rs b/crates/jmap/src/services/delivery.rs index 23cccb69..0bb0f10b 100644 --- a/crates/jmap/src/services/delivery.rs +++ b/crates/jmap/src/services/delivery.rs @@ -21,19 +21,19 @@ * for more details. */ -use std::sync::Arc; - +use common::DeliveryEvent; use tokio::sync::mpsc; -use utils::ipc::DeliveryEvent; -use crate::JMAP; +use crate::{JmapInstance, JMAP}; -pub fn spawn_delivery_manager(core: Arc, mut delivery_rx: mpsc::Receiver) { +pub fn spawn_delivery_manager(core: JmapInstance, mut delivery_rx: mpsc::Receiver) { tokio::spawn(async move { while let Some(event) = delivery_rx.recv().await { match event { DeliveryEvent::Ingest { message, result_tx } => { - result_tx.send(core.deliver_message(message).await).ok(); + result_tx + .send(JMAP::from(core.clone()).deliver_message(message).await) + .ok(); } DeliveryEvent::Stop => break, } diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index e53da2ac..b363ab68 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -21,24 +21,17 @@ * for more details. */ -use std::sync::Arc; +use std::{collections::BinaryHeap, time::Instant}; -use store::dispatch::blocked::BLOCKED_IP_PREFIX; +use store::write::purge::PurgeStore; use tokio::sync::mpsc; -use utils::{ - config::{cron::SimpleCron, Config, Servers}, - map::ttl_dashmap::TtlMap, - UnwrapFailure, -}; +use utils::map::ttl_dashmap::TtlMap; -use crate::JMAP; +use crate::{Inner, JmapInstance, JMAP, LONG_SLUMBER}; use super::IPC_CHANNEL_BUFFER; pub enum Event { - PurgeSessions, - ReloadCertificates, - ReloadConfig, IndexStart, IndexDone, #[cfg(feature = "test_mode")] @@ -46,18 +39,19 @@ pub enum Event { Exit, } -pub fn spawn_housekeeper( - core: Arc, - settings: &Config, - servers: &mut Servers, - mut rx: mpsc::Receiver, -) { - let purge_cache = settings - .property_or_default::("jmap.session.purge.frequency", "15 * *") - .failed("Initialize housekeeper"); +#[derive(PartialEq, Eq)] +struct PurgeEvent { + due: Instant, + event: PurgeClass, +} - let certificates = std::mem::take(&mut servers.certificates); +#[derive(PartialEq, Eq)] +enum PurgeClass { + Session, + Store(usize), +} +pub fn spawn_housekeeper(core: JmapInstance, mut rx: mpsc::Receiver) { tokio::spawn(async move { tracing::debug!("Housekeeper task started."); @@ -65,84 +59,39 @@ pub fn spawn_housekeeper( let mut index_pending = false; // Index any queued messages - let core_ = core.clone(); + let jmap = JMAP::from(core.clone()); tokio::spawn(async move { - core_.fts_index_queued().await; + jmap.fts_index_queued().await; }); + let mut heap = BinaryHeap::new(); + + // Add all purge events to heap + let core_ = core.core.load(); + heap.push(PurgeEvent { + due: Instant::now() + core_.jmap.session_purge_frequency.time_to_next(), + event: PurgeClass::Session, + }); + for (idx, schedule) in core_.storage.purge_schedules.iter().enumerate() { + heap.push(PurgeEvent { + due: Instant::now() + schedule.cron.time_to_next(), + event: PurgeClass::Store(idx), + }); + } loop { - let time_to_next = purge_cache.time_to_next(); - let mut do_purge = false; + let time_to_next = heap + .peek() + .map(|e| e.due.saturating_duration_since(Instant::now())) + .unwrap_or(LONG_SLUMBER); match tokio::time::timeout(time_to_next, rx.recv()).await { Ok(Some(event)) => match event { - Event::PurgeSessions => { - do_purge = true; - } - Event::ReloadCertificates => { - let certificates = certificates.clone(); - tokio::spawn(async move { - for cert in certificates { - match cert.reload().await { - Ok(_) => { - tracing::info!( - context = "tls", - event = "reload", - path = cert.path[0].to_string_lossy().as_ref(), - "Reloaded certificate." - ); - } - Err(err) => { - tracing::error!( - context = "tls", - event = "error", - path = cert.path[0].to_string_lossy().as_ref(), - error = ?err, - "Failed to reload certificate." - ); - } - } - } - }); - } - Event::ReloadConfig => { - // Future releases will support reloading the configuration - // for now, we just reload the blocked IP addresses - let core = core.clone(); - let todo = "fix"; - /*tokio::spawn(async move { - match core.store.config_list(BLOCKED_IP_PREFIX, true).await { - Ok(settings) => { - if let Err(err) = core - .directory - .blocked_ips - .reload_blocked_ips(settings.iter().map(|(k, _)| k)) - { - tracing::error!( - context = "store", - event = "error", - error = ?err, - "Failed to reload configuration." - ); - } - } - Err(err) => { - tracing::error!( - context = "store", - event = "error", - error = ?err, - "Failed to reload configuration." - ); - } - } - });*/ - } Event::IndexStart => { if !index_busy { index_busy = true; - let core = core.clone(); + let jmap = JMAP::from(core.clone()); tokio::spawn(async move { - core.fts_index_queued().await; + jmap.fts_index_queued().await; }); } else { index_pending = true; @@ -151,9 +100,9 @@ pub fn spawn_housekeeper( Event::IndexDone => { if index_pending { index_pending = false; - let core = core.clone(); + let jmap = JMAP::from(core.clone()); tokio::spawn(async move { - core.fts_index_queued().await; + jmap.fts_index_queued().await; }); } else { index_busy = false; @@ -173,25 +122,93 @@ pub fn spawn_housekeeper( return; } Err(_) => { - do_purge = true; - } - } + let core_ = core.core.load(); + while let Some(event) = heap.peek() { + if event.due > Instant::now() { + break; + } + let event = heap.pop().unwrap(); + match event.event { + PurgeClass::Session => { + let inner = core.jmap_inner.clone(); + tokio::spawn(async move { + tracing::debug!("Purging session cache."); + inner.purge(); + }); + heap.push(PurgeEvent { + due: Instant::now() + + core_.jmap.session_purge_frequency.time_to_next(), + event: PurgeClass::Session, + }); + } + PurgeClass::Store(idx) => { + if let Some(schedule) = + core_.storage.purge_schedules.get(idx).cloned() + { + heap.push(PurgeEvent { + due: Instant::now() + schedule.cron.time_to_next(), + event: PurgeClass::Store(idx), + }); + tokio::spawn(async move { + let (class, result) = match schedule.store { + PurgeStore::Data(store) => { + ("data", store.purge_store().await) + } + PurgeStore::Blobs { store, blob_store } => { + ("blob", store.purge_blobs(blob_store).await) + } + PurgeStore::Lookup(lookup_store) => { + ("lookup", lookup_store.purge_lookup_store().await) + } + }; - if do_purge { - let core = core.clone(); - tokio::spawn(async move { - tracing::info!("Purging session cache."); - core.sessions.cleanup(); - core.access_tokens.cleanup(); - core.oauth_codes.cleanup(); - core.concurrency_limiter - .retain(|_, limiter| limiter.is_active()); - }); + match result { + Ok(_) => { + tracing::debug!( + "Purged {class} store {}.", + schedule.store_id + ); + } + Err(err) => { + tracing::error!( + "Failed to purge {class} store {}: {err}", + schedule.store_id + ); + } + } + }); + } + } + } + } + } } } }); } +impl Ord for PurgeEvent { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.due.cmp(&other.due).reverse() + } +} + +impl PartialOrd for PurgeEvent { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Inner { + pub fn purge(&self) { + self.sessions.cleanup(); + self.access_tokens.cleanup(); + self.oauth_codes.cleanup(); + self.concurrency_limiter + .retain(|_, limiter| limiter.is_active()); + } +} + pub fn init_housekeeper() -> (mpsc::Sender, mpsc::Receiver) { mpsc::channel::(IPC_CHANNEL_BUFFER) } diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index 455d95a3..c7d6d67b 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -61,7 +61,9 @@ impl JMAP { // TODO: Support indexing from multiple nodes let mut entries = Vec::new(); let _ = self - .store + .core + .storage + .data .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { @@ -115,12 +117,12 @@ impl JMAP { // Index message let document = - FtsDocument::with_default_language(self.config.default_language) + FtsDocument::with_default_language(self.core.jmap.default_language) .with_account_id(key.account_id) .with_collection(Collection::Email) .with_document_id(key.document_id) .index_message(&message); - if let Err(err) = self.fts_store.index(document).await { + if let Err(err) = self.core.storage.fts.index(document).await { tracing::error!( context = "fts_index_queued", event = "error", @@ -165,7 +167,9 @@ impl JMAP { } } else { if let Err(err) = self - .fts_store + .core + .storage + .fts .remove(key.account_id, Collection::Email.into(), key.document_id) .await { @@ -191,7 +195,9 @@ impl JMAP { // Remove entry from queue if let Err(err) = self - .store + .core + .storage + .data .write( BatchBuilder::new() .with_account_id(key.account_id) @@ -211,7 +217,7 @@ impl JMAP { } } - if let Err(err) = self.housekeeper_tx.send(Event::IndexDone).await { + if let Err(err) = self.inner.housekeeper_tx.send(Event::IndexDone).await { tracing::warn!("Failed to send index done event to housekeeper: {}", err); } } diff --git a/crates/jmap/src/services/ingest.rs b/crates/jmap/src/services/ingest.rs index fb97e8bf..aeefc8a7 100644 --- a/crates/jmap/src/services/ingest.rs +++ b/crates/jmap/src/services/ingest.rs @@ -21,11 +21,11 @@ * for more details. */ +use common::{DeliveryResult, IngestMessage}; use directory::QueryBy; use jmap_proto::types::{state::StateChange, type_state::DataType}; use mail_parser::MessageParser; use store::ahash::AHashMap; -use utils::ipc::{DeliveryResult, IngestMessage}; use crate::{email::ingest::IngestEmail, mailbox::INBOX_ID, IngestError, JMAP}; @@ -33,7 +33,9 @@ impl JMAP { pub async fn deliver_message(&self, message: IngestMessage) -> Vec { // Read message let raw_message = match self - .blob_store + .core + .storage + .blob .get_blob(message.message_blob.as_slice(), 0..usize::MAX) .await { @@ -51,7 +53,11 @@ impl JMAP { let mut recipients = Vec::with_capacity(message.recipients.len()); let mut deliver_names = AHashMap::with_capacity(message.recipients.len()); for rcpt in &message.recipients { - let uids = self.directory.email_to_ids(rcpt).await.unwrap_or_default(); + let uids = self + .core + .email_to_ids(&self.core.storage.directory, rcpt) + .await + .unwrap_or_default(); for uid in &uids { deliver_names.insert(*uid, (DeliveryResult::Success, rcpt)); } @@ -73,7 +79,13 @@ impl JMAP { .await } Ok(None) => { - let account_quota = match self.directory.query(QueryBy::Id(*uid), false).await { + let account_quota = match self + .core + .storage + .directory + .query(QueryBy::Id(*uid), false) + .await + { Ok(Some(p)) => p.quota as i64, Ok(None) => 0, Err(_) => { @@ -93,7 +105,7 @@ impl JMAP { keywords: vec![], received_at: None, skip_duplicates: true, - encrypt: self.config.encrypt, + encrypt: self.core.jmap.encrypt, }) .await } diff --git a/crates/jmap/src/services/state.rs b/crates/jmap/src/services/state.rs index 698c2320..850ce6d1 100644 --- a/crates/jmap/src/services/state.rs +++ b/crates/jmap/src/services/state.rs @@ -21,19 +21,16 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant, SystemTime}, -}; +use std::time::{Duration, Instant, SystemTime}; use jmap_proto::types::{id::Id, state::StateChange, type_state::DataType}; use store::ahash::AHashMap; use tokio::sync::mpsc; -use utils::{config::Config, map::bitmap::Bitmap}; +use utils::map::bitmap::Bitmap; use crate::{ push::{manager::spawn_push_manager, UpdateSubscription}, - JMAP, + JmapInstance, JMAP, }; use super::IPC_CHANNEL_BUFFER; @@ -93,12 +90,8 @@ enum SubscriberId { } #[allow(clippy::unwrap_or_default)] -pub fn spawn_state_manager( - core: Arc, - settings: &Config, - mut change_rx: mpsc::Receiver, -) { - let push_tx = spawn_push_manager(settings); +pub fn spawn_state_manager(core: JmapInstance, mut change_rx: mpsc::Receiver) { + let push_tx = spawn_push_manager(core.clone()); tokio::spawn(async move { let mut subscribers: AHashMap> = @@ -121,7 +114,7 @@ pub fn spawn_state_manager( } Event::UpdateSharedAccounts { account_id } => { // Obtain account membership and shared mailboxes - let acl = match core.get_access_token(account_id).await { + let acl = match JMAP::from(core.clone()).get_access_token(account_id).await { Some(result) => result, None => { continue; @@ -397,7 +390,7 @@ impl JMAP { types: Bitmap, ) -> Option> { let (change_tx, change_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); - let state_tx = self.state_tx.clone(); + let state_tx = self.inner.state_tx.clone(); for event in [ Event::UpdateSharedAccounts { account_id }, @@ -421,6 +414,7 @@ impl JMAP { pub async fn broadcast_state_change(&self, state_change: StateChange) -> bool { match self + .inner .state_tx .clone() .send(Event::Publish { state_change }) @@ -446,7 +440,7 @@ impl JMAP { } }; - let state_tx = self.state_tx.clone(); + let state_tx = self.inner.state_tx.clone(); for event in [Event::UpdateSharedAccounts { account_id }, push_subs] { if let Err(err) = state_tx.send(event).await { tracing::error!("Channel failure while publishing state change: {}", err); diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 530b9cde..8a2a1a7b 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -45,7 +45,7 @@ impl JMAP { &self, mut request: GetRequest, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[Property::Id, Property::Name, Property::BlobId]); let account_id = request.account_id.document_id(); @@ -58,7 +58,7 @@ impl JMAP { } else { push_ids .iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; @@ -238,9 +238,8 @@ impl JMAP { Ok((sieve.inner, script_object.inner)) } else { // Deserialization failed, probably because the script compiler version changed - match self - .sieve_compiler - .compile(script_bytes.get(0..script_offset).ok_or_else(|| { + match self.core.sieve.untrusted_compiler.compile( + script_bytes.get(0..script_offset).ok_or_else(|| { tracing::warn!( context = "sieve_script_compile", event = "error", @@ -250,7 +249,8 @@ impl JMAP { ); MethodError::ServerPartialFail - })?) { + })?, + ) { Ok(sieve) => { // Store updated compiled sieve script let sieve = Bincode::new(sieve); diff --git a/crates/jmap/src/sieve/ingest.rs b/crates/jmap/src/sieve/ingest.rs index e7d4276a..955c914c 100644 --- a/crates/jmap/src/sieve/ingest.rs +++ b/crates/jmap/src/sieve/ingest.rs @@ -23,6 +23,7 @@ use std::borrow::Cow; +use common::listener::stream::NullIo; use directory::QueryBy; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; use mail_parser::MessageParser; @@ -32,7 +33,6 @@ use store::{ ahash::AHashSet, write::{now, BatchBuilder, Bincode, F_VALUE}, }; -use utils::listener::stream::NullIo; use crate::{ email::ingest::{IngestEmail, IngestedEmail}, @@ -76,20 +76,25 @@ impl JMAP { .map_err(|_| IngestError::Temporary)?; // Create Sieve instance - let mut instance = self.sieve_runtime.filter_parsed(message); + let mut instance = self.core.sieve.untrusted_runtime.filter_parsed(message); // Set account name and obtain quota - let (account_quota, mail_from) = - match self.directory.query(QueryBy::Id(account_id), false).await { - Ok(Some(p)) => { - instance.set_user_full_name(p.description().unwrap_or_else(|| p.name())); - (p.quota as i64, p.emails.into_iter().next()) - } - Ok(None) => (0, None), - Err(_) => { - return Err(IngestError::Temporary); - } - }; + let (account_quota, mail_from) = match self + .core + .storage + .directory + .query(QueryBy::Id(account_id), false) + .await + { + Ok(Some(p)) => { + instance.set_user_full_name(p.description().unwrap_or_else(|| p.name())); + (p.quota as i64, p.emails.into_iter().next()) + } + Ok(None) => (0, None), + Err(_) => { + return Err(IngestError::Temporary); + } + }; // Set account address let mail_from = mail_from.unwrap_or_else(|| envelope_to.to_string()); @@ -328,7 +333,7 @@ impl JMAP { } => { input = true.into(); if let Some(message) = messages.get(message_id) { - if message.raw_message.len() <= self.config.mail_max_size { + if message.raw_message.len() <= self.core.jmap.mail_max_size { let result = Session::::sieve( self.smtp.clone(), SessionAddress::new(mail_from.clone()), @@ -358,7 +363,7 @@ impl JMAP { event = "message_too_large", from = mail_from.as_str(), size = message.raw_message.len(), - max_size = self.config.mail_max_size + max_size = self.core.jmap.mail_max_size ); } } else { @@ -442,7 +447,7 @@ impl JMAP { keywords: sieve_message.flags, received_at: None, skip_duplicates: true, - encrypt: self.config.encrypt, + encrypt: self.core.jmap.encrypt, }) .await { diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 6a37d161..c2f254d6 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -97,7 +97,7 @@ impl JMAP { // Process creates let mut changes = ChangeLogBuilder::new(); for (id, object) in request.unwrap_create() { - if sieve_ids.len() as usize <= self.config.sieve_max_scripts { + if sieve_ids.len() as usize <= self.core.jmap.sieve_max_scripts { match self.sieve_set_item(object, None, &ctx).await? { Ok((mut builder, Some(blob))) => { // Obtain document id @@ -248,7 +248,7 @@ impl JMAP { if !batch.is_empty() { changes.log_update(Collection::SieveScript, document_id); - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => (), Err(store::Error::AssertValueFailed) => { ctx.response.not_updated.append(id, SetError::forbidden().with_description( @@ -443,7 +443,7 @@ impl JMAP { }; let value = match (&property, value) { (Property::Name, MaybePatchValue::Value(Value::Text(value))) => { - if value.len() > self.config.sieve_max_script_name { + if value.len() > self.core.jmap.sieve_max_script_name { return Ok(Err(SetError::invalid_properties() .with_property(property) .with_description("Script name is too long."))); @@ -533,7 +533,7 @@ impl JMAP { } // Compile script - match self.sieve_compiler.compile(&bytes) { + match self.core.sieve.untrusted_compiler.compile(&bytes) { Ok(script) => { changes.set(Property::BlobId, BlobId::default().with_section_size(bytes.len())); bytes.extend(bincode::serialize(&script).unwrap_or_default()); @@ -654,7 +654,7 @@ impl JMAP { // Write changes if !changed_ids.is_empty() { - match self.store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => (), Err(store::Error::AssertValueFailed) => { return Ok(vec![]); diff --git a/crates/jmap/src/sieve/validate.rs b/crates/jmap/src/sieve/validate.rs index d2333f68..a7dc776b 100644 --- a/crates/jmap/src/sieve/validate.rs +++ b/crates/jmap/src/sieve/validate.rs @@ -42,7 +42,7 @@ impl JMAP { error: match self .blob_download(&request.blob_id, access_token) .await? - .map(|bytes| self.sieve_compiler.compile(&bytes)) + .map(|bytes| self.core.sieve.untrusted_compiler.compile(&bytes)) { Some(Ok(_)) => None, Some(Err(err)) => SetError::new(SetErrorType::InvalidScript) diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index 39e58727..102ae206 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -36,7 +36,7 @@ impl JMAP { &self, mut request: GetRequest, ) -> Result { - let ids = request.unwrap_ids(self.config.get_max_objects)?; + let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; let properties = request.unwrap_properties(&[ Property::Id, Property::EmailId, @@ -59,7 +59,7 @@ impl JMAP { } else { email_submission_ids .iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 98be8f02..68ad947d 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -23,6 +23,7 @@ use std::{collections::HashMap, sync::Arc}; +use common::listener::{stream::NullIo, ServerInstance}; use jmap_proto::{ error::{ method::MethodError, @@ -51,10 +52,7 @@ use mail_parser::{HeaderName, HeaderValue}; use smtp::core::{Session, SessionData, State}; use smtp_proto::{request::parser::Rfc5321Parser, MailFrom, RcptTo}; use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder, Bincode}; -use utils::{ - listener::{stream::NullIo, ServerInstance}, - map::vec_map::VecMap, -}; +use utils::map::vec_map::VecMap; use crate::{email::metadata::MessageMetadata, identity::set::sanitize_email, JMAP}; @@ -77,7 +75,7 @@ impl JMAP { next_call: &mut Option>, ) -> Result { let account_id = request.account_id.document_id(); - let mut response = SetResponse::from_request(&request, self.config.set_max_objects)?; + let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy(); // Process creates @@ -561,11 +559,11 @@ impl JMAP { // Obtain raw message let message = if let Some(message) = self.get_blob(&metadata.blob_hash, 0..usize::MAX).await? { - if message.len() > self.config.mail_max_size { + if message.len() > self.core.jmap.mail_max_size { return Ok(Err(SetError::new(SetErrorType::InvalidEmail) .with_description(format!( "Message exceeds maximum size of {} bytes.", - self.config.mail_max_size + self.core.jmap.mail_max_size )))); } diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 5b8cc1b1..5396d29b 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -37,14 +37,14 @@ impl JMAP { mut request: GetRequest, ) -> Result { let account_id = request.account_id.document_id(); - let ids = if let Some(ids) = request.unwrap_ids(self.config.get_max_objects)? { + let ids = if let Some(ids) = request.unwrap_ids(self.core.jmap.get_max_objects)? { ids } else { self.get_document_ids(account_id, Collection::Thread) .await? .unwrap_or_default() .into_iter() - .take(self.config.get_max_objects) + .take(self.core.jmap.get_max_objects) .map(Into::into) .collect() }; @@ -68,7 +68,9 @@ impl JMAP { if add_email_ids { thread.append( Property::EmailIds, - self.store + self.core + .storage + .data .sort( ResultSet::new(account_id, Collection::Email, document_ids.clone()), vec![Comparator::ascending(Property::ReceivedAt)], diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 5ec1c8c4..a9730da5 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -436,7 +436,7 @@ impl JMAP { script.extend_from_slice(b"}\r\n"); } - match self.sieve_compiler.compile(&script) { + match self.core.sieve.untrusted_compiler.compile(&script) { Ok(compiled_script) => { // Update blob length obj.set( diff --git a/crates/jmap/src/websocket/stream.rs b/crates/jmap/src/websocket/stream.rs index 0915e689..2e6d4669 100644 --- a/crates/jmap/src/websocket/stream.rs +++ b/crates/jmap/src/websocket/stream.rs @@ -23,6 +23,7 @@ use std::{sync::Arc, time::Instant}; +use common::listener::ServerInstance; use futures_util::{SinkExt, StreamExt}; use hyper::upgrade::Upgraded; use hyper_util::rt::TokioIo; @@ -35,7 +36,7 @@ use jmap_proto::{ }; use tokio_tungstenite::WebSocketStream; use tungstenite::Message; -use utils::{listener::ServerInstance, map::bitmap::Bitmap}; +use utils::map::bitmap::Bitmap; use crate::{auth::AccessToken, JMAP}; @@ -49,13 +50,12 @@ impl JMAP { let span = tracing::info_span!( "WebSocket connection established", "account_id" = access_token.primary_id(), - "url" = instance.data, ); // Set timeouts - let throttle = self.config.web_socket_throttle; - let timeout = self.config.web_socket_timeout; - let heartbeat = self.config.web_socket_heartbeat; + let throttle = self.core.jmap.web_socket_throttle; + let timeout = self.core.jmap.web_socket_timeout; + let heartbeat = self.core.jmap.web_socket_heartbeat; let mut last_request = Instant::now(); let mut last_changes_sent = Instant::now() - throttle; let mut last_heartbeat = Instant::now() - heartbeat; @@ -87,8 +87,8 @@ impl JMAP { Message::Text(text) => { let response = match WebSocketMessage::parse( text.as_bytes(), - self.config.request_max_calls, - self.config.request_max_size, + self.core.jmap.request_max_calls, + self.core.jmap.request_max_size, ) { Ok(WebSocketMessage::Request(request)) => { match self diff --git a/crates/jmap/src/websocket/upgrade.rs b/crates/jmap/src/websocket/upgrade.rs index 5516ebf1..5bdc654a 100644 --- a/crates/jmap/src/websocket/upgrade.rs +++ b/crates/jmap/src/websocket/upgrade.rs @@ -23,13 +23,13 @@ use std::sync::Arc; +use common::listener::ServerInstance; use http_body_util::{BodyExt, Full}; use hyper::{body::Bytes, Response, StatusCode}; use hyper_util::rt::TokioIo; use jmap_proto::error::request::RequestError; use tokio_tungstenite::WebSocketStream; use tungstenite::{handshake::derive_accept_key, protocol::Role}; -use utils::listener::ServerInstance; use crate::{ api::{http::ToHttpResponse, HttpRequest, HttpResponse}, @@ -37,77 +37,84 @@ use crate::{ JMAP, }; -pub async fn upgrade_websocket_connection( - jmap: Arc, - req: HttpRequest, - access_token: Arc, - instance: Arc, -) -> HttpResponse { - let headers = req.headers(); - if headers - .get(hyper::header::CONNECTION) - .and_then(|h| h.to_str().ok()) - != Some("Upgrade") - || headers - .get(hyper::header::UPGRADE) +impl JMAP { + pub async fn upgrade_websocket_connection( + &self, + req: HttpRequest, + access_token: Arc, + instance: Arc, + ) -> HttpResponse { + let headers = req.headers(); + if headers + .get(hyper::header::CONNECTION) .and_then(|h| h.to_str().ok()) - != Some("websocket") - { - return RequestError::blank( - StatusCode::BAD_REQUEST.as_u16(), - "WebSocket upgrade failed", - "Missing or Invalid Connection or Upgrade headers.", - ) - .into_http_response(); - } - let derived_key = match ( - headers - .get("Sec-WebSocket-Key") - .and_then(|h| h.to_str().ok()), - headers - .get("Sec-WebSocket-Version") - .and_then(|h| h.to_str().ok()), - ) { - (Some(key), Some("13")) => derive_accept_key(key.as_bytes()), - _ => { + != Some("Upgrade") + || headers + .get(hyper::header::UPGRADE) + .and_then(|h| h.to_str().ok()) + != Some("websocket") + { return RequestError::blank( StatusCode::BAD_REQUEST.as_u16(), "WebSocket upgrade failed", - "Missing or Invalid Sec-WebSocket-Key headers.", + "Missing or Invalid Connection or Upgrade headers.", ) .into_http_response(); } - }; - - // Spawn WebSocket connection - tokio::spawn(async move { - // Upgrade connection - match hyper::upgrade::on(req).await { - Ok(upgraded) => { - jmap.handle_websocket_stream( - WebSocketStream::from_raw_socket(TokioIo::new(upgraded), Role::Server, None) - .await, - access_token, - instance, + let derived_key = match ( + headers + .get("Sec-WebSocket-Key") + .and_then(|h| h.to_str().ok()), + headers + .get("Sec-WebSocket-Version") + .and_then(|h| h.to_str().ok()), + ) { + (Some(key), Some("13")) => derive_accept_key(key.as_bytes()), + _ => { + return RequestError::blank( + StatusCode::BAD_REQUEST.as_u16(), + "WebSocket upgrade failed", + "Missing or Invalid Sec-WebSocket-Key headers.", ) - .await; + .into_http_response(); } - Err(e) => { - tracing::debug!("WebSocket upgrade failed: {}", e); - } - } - }); + }; - Response::builder() - .status(hyper::StatusCode::SWITCHING_PROTOCOLS) - .header(hyper::header::CONNECTION, "upgrade") - .header(hyper::header::UPGRADE, "websocket") - .header("Sec-WebSocket-Accept", &derived_key) - .header("Sec-WebSocket-Protocol", "jmap") - .body( - Full::new(Bytes::from("Switching to WebSocket protocol")) - .map_err(|never| match never {}) - .boxed(), - ) - .unwrap() + // Spawn WebSocket connection + let jmap = self.clone(); + tokio::spawn(async move { + // Upgrade connection + match hyper::upgrade::on(req).await { + Ok(upgraded) => { + jmap.handle_websocket_stream( + WebSocketStream::from_raw_socket( + TokioIo::new(upgraded), + Role::Server, + None, + ) + .await, + access_token, + instance, + ) + .await; + } + Err(e) => { + tracing::debug!("WebSocket upgrade failed: {}", e); + } + } + }); + + Response::builder() + .status(hyper::StatusCode::SWITCHING_PROTOCOLS) + .header(hyper::header::CONNECTION, "upgrade") + .header(hyper::header::UPGRADE, "websocket") + .header("Sec-WebSocket-Accept", &derived_key) + .header("Sec-WebSocket-Protocol", "jmap") + .body( + Full::new(Bytes::from("Switching to WebSocket protocol")) + .map_err(|never| match never {}) + .boxed(), + ) + .unwrap() + } } diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 359c9635..d165d288 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -22,6 +22,7 @@ jmap_proto = { path = "../jmap-proto" } smtp = { path = "../smtp", features = ["local_delivery"] } imap = { path = "../imap" } managesieve = { path = "../managesieve" } +common = { path = "../common" } directory = { path = "../directory" } utils = { path = "../utils" } tokio = { version = "1.23", features = ["full"] } diff --git a/crates/main/src/main.rs b/crates/main/src/main.rs index e0a32a30..c63c0bbf 100644 --- a/crates/main/src/main.rs +++ b/crates/main/src/main.rs @@ -23,16 +23,22 @@ use std::time::Duration; -use directory::core::config::ConfigDirectory; +use common::{ + config::{ + server::{ServerProtocol, Servers}, + tracers::Tracers, + }, + Core, +}; use imap::core::{ImapSessionManager, IMAP}; use jmap::{api::JmapSessionManager, services::IPC_CHANNEL_BUFFER, JMAP}; use managesieve::core::ManageSieveSessionManager; use smtp::core::{SmtpSessionManager, SMTP}; -use store::config::ConfigStore; +use store::Stores; use tokio::sync::mpsc; use utils::{ - config::{Config, ServerProtocol}, - enable_tracing, wait_for_shutdown, UnwrapFailure, + config::{Config, ConfigError}, + wait_for_shutdown, }; #[cfg(not(target_env = "msvc"))] @@ -44,100 +50,76 @@ static GLOBAL: Jemalloc = Jemalloc; #[tokio::main] async fn main() -> std::io::Result<()> { + // Load config and apply macros let mut config = Config::init(); + config.resolve_macros().await; - // Enable tracing - let _tracer = enable_tracing( - &config, - &format!( - "Starting Stalwart Mail Server v{}...", - env!("CARGO_PKG_VERSION"), - ), - ) - .failed("Failed to enable tracing"); + // Parse servers + let servers = Servers::parse(&mut config); // Bind ports and drop privileges - let mut servers = config.parse_servers().failed("Invalid configuration"); - servers.bind(&config); + servers.bind_and_drop_priv(&mut config); - // Parse stores - let stores = config.parse_stores().await.failed("Invalid configuration"); - let data_store = stores - .get_store(&config, "storage.data") - .failed("Invalid configuration"); + // Build stores + let stores = Stores::parse(&mut config).await; + let todo = "merge config with data store, resolve macros"; - // Update configuration - config.update( - data_store - .config_list("", false) - .await - .failed("Storage error"), + // Enable tracing + let guards = Tracers::parse(&mut config).enable(&mut config); + tracing::info!( + "Starting Stalwart Mail Server v{}...", + env!("CARGO_PKG_VERSION") ); - // Parse directories - let directory = config - .parse_directory(&stores, data_store) - .await - .failed("Invalid configuration"); - let schedulers = config - .parse_purge_schedules( - &stores, - config.value("storage.data"), - config.value("storage.blob"), - ) - .await - .failed("Invalid configuration"); + // Parse core + let core = Core::parse(&mut config, stores).await; + let store = core.storage.data.clone(); + let shared_core = core.into_shared(); // Init servers let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let smtp = SMTP::init(&config, &servers, &stores, &directory, delivery_tx) - .await - .failed("Invalid configuration file"); + let smtp = SMTP::init(&mut config, shared_core.clone(), delivery_tx).await; let jmap = JMAP::init( - &config, - &stores, - &directory, - &mut servers, + &mut config, delivery_rx, - smtp.clone(), + shared_core.clone(), + smtp.inner.clone(), ) - .await - .failed("Invalid configuration file"); - let imap = IMAP::init(&config) - .await - .failed("Invalid configuration file"); - jmap.directory - .blocked_ips - .reload(&config) - .failed("Invalid configuration"); + .await; + let imap = IMAP::init(&mut config, jmap.clone()).await; + + // Log configuration errors + config.log_errors(guards.is_none()); + config.log_warnings(guards.is_none()); // Spawn servers - let (shutdown_tx, shutdown_rx) = servers.spawn(|server, shutdown_rx| { - match &server.protocol { - ServerProtocol::Smtp | ServerProtocol::Lmtp => { - server.spawn(SmtpSessionManager::new(smtp.clone()), shutdown_rx) - } - ServerProtocol::Http => { - tracing::debug!("Ignoring HTTP server listener, using JMAP port instead."); - } - ServerProtocol::Jmap => { - server.spawn(JmapSessionManager::new(jmap.clone()), shutdown_rx) - } - ServerProtocol::Imap => server.spawn( - ImapSessionManager::new(jmap.clone(), imap.clone()), - shutdown_rx, - ), - ServerProtocol::ManageSieve => server.spawn( - ManageSieveSessionManager::new(jmap.clone(), imap.clone()), - shutdown_rx, - ), - }; - }); - - // Spawn purge schedulers - for scheduler in schedulers { - scheduler.spawn(shutdown_rx.clone()); - } + let shutdown_tx = servers.spawn( + |server, shutdown_rx| { + match &server.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( + SmtpSessionManager::new(smtp.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::Http => server.spawn( + JmapSessionManager::new(jmap.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::Imap => server.spawn( + ImapSessionManager::new(imap.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::ManageSieve => server.spawn( + ManageSieveSessionManager::new(imap.clone()), + shared_core.clone(), + shutdown_rx, + ), + }; + }, + store, + ); // Wait for shutdown signal wait_for_shutdown(&format!( diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index 79f8ec95..7d1ae127 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -10,6 +10,7 @@ imap = { path = "../imap" } jmap = { path = "../jmap" } jmap_proto = { path = "../jmap-proto" } directory = { path = "../directory" } +common = { path = "../common" } store = { path = "../store" } utils = { path = "../utils" } mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } diff --git a/crates/managesieve/src/core/client.rs b/crates/managesieve/src/core/client.rs index 77373613..e0a50144 100644 --- a/crates/managesieve/src/core/client.rs +++ b/crates/managesieve/src/core/client.rs @@ -21,11 +21,11 @@ * for more details. */ +use common::listener::SessionStream; use imap_proto::receiver::{self, Request}; use jmap_proto::types::{collection::Collection, property::Property}; use store::query::Filter; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use utils::listener::SessionStream; use super::{Command, ResponseCode, ResponseType, Session, State, StatusResponse}; @@ -114,7 +114,7 @@ impl Session { Command::Capability | Command::Logout | Command::Noop => Ok(command), Command::Authenticate => { if let State::NotAuthenticated { .. } = &self.state { - if self.stream.is_tls() || self.imap.allow_plain_auth { + if self.stream.is_tls() || self.jmap.core.imap.allow_plain_auth { Ok(command) } else { Err(StatusResponse::no("Cannot authenticate over plain-text.") @@ -141,21 +141,27 @@ impl Session { | Command::CheckScript | Command::Unauthenticate => { if let State::Authenticated { access_token, .. } = &self.state { - match self - .jmap - .lookup_store - .is_rate_allowed( - format!("ireq:{}", access_token.primary_id()).as_bytes(), - &self.imap.rate_requests, - true, - ) - .await - { - Ok(None) => Ok(command), - Ok(Some(_)) => Err(StatusResponse::no("Too many requests") - .with_code(ResponseCode::TryLater)), - Err(_) => Err(StatusResponse::no("Internal server error") - .with_code(ResponseCode::TryLater)), + if let Some(rate) = &self.jmap.core.imap.rate_requests { + match self + .jmap + .core + .storage + .lookup + .is_rate_allowed( + format!("ireq:{}", access_token.primary_id()).as_bytes(), + rate, + true, + ) + .await + { + Ok(None) => Ok(command), + Ok(Some(_)) => Err(StatusResponse::no("Too many requests") + .with_code(ResponseCode::TryLater)), + Err(_) => Err(StatusResponse::no("Internal server error") + .with_code(ResponseCode::TryLater)), + } + } else { + Ok(command) } } else { Err(StatusResponse::no("Not authenticated.")) @@ -216,7 +222,9 @@ impl Session { impl Session { pub async fn get_script_id(&self, account_id: u32, name: &str) -> Result { self.jmap - .store + .core + .storage + .data .filter( account_id, Collection::SieveScript, diff --git a/crates/managesieve/src/core/mod.rs b/crates/managesieve/src/core/mod.rs index 5c9a2e64..c6ea872e 100644 --- a/crates/managesieve/src/core/mod.rs +++ b/crates/managesieve/src/core/mod.rs @@ -26,15 +26,15 @@ pub mod session; use std::{borrow::Cow, net::IpAddr, sync::Arc}; -use imap::core::IMAP; +use common::listener::{limiter::InFlight, ServerInstance}; +use imap::core::{ImapInstance, Inner}; use imap_proto::receiver::{CommandParser, Receiver}; use jmap::{auth::AccessToken, JMAP}; use tokio::io::{AsyncRead, AsyncWrite}; -use utils::listener::{limiter::InFlight, ServerInstance}; pub struct Session { - pub jmap: Arc, - pub imap: Arc, + pub jmap: JMAP, + pub imap: Arc, pub instance: Arc, pub receiver: Receiver, pub state: State, @@ -50,7 +50,7 @@ pub enum State { }, Authenticated { access_token: Arc, - in_flight: InFlight, + in_flight: Option, }, } @@ -65,13 +65,12 @@ impl State { #[derive(Clone)] pub struct ManageSieveSessionManager { - pub jmap: Arc, - pub imap: Arc, + pub imap: ImapInstance, } impl ManageSieveSessionManager { - pub fn new(jmap: Arc, imap: Arc) -> Self { - Self { jmap, imap } + pub fn new(imap: ImapInstance) -> Self { + Self { imap } } } diff --git a/crates/managesieve/src/core/session.rs b/crates/managesieve/src/core/session.rs index c7c27f41..4b102108 100644 --- a/crates/managesieve/src/core/session.rs +++ b/crates/managesieve/src/core/session.rs @@ -21,9 +21,10 @@ * for more details. */ +use common::listener::{SessionData, SessionManager, SessionStream}; use imap_proto::receiver::{self, Receiver}; +use jmap::JMAP; use tokio_rustls::server::TlsStream; -use utils::listener::{SessionManager, SessionStream}; use crate::SERVER_GREETING; @@ -33,15 +34,16 @@ impl SessionManager for ManageSieveSessionManager { #[allow(clippy::manual_async_fn)] fn handle( self, - session: utils::listener::SessionData, + session: SessionData, ) -> impl std::future::Future + Send { async move { // Create session + let jmap = JMAP::from(self.imap.jmap_instance); let mut session = Session { - receiver: Receiver::with_max_request_size(self.imap.max_request_size) + receiver: Receiver::with_max_request_size(jmap.core.imap.max_request_size) .with_start_state(receiver::State::Command { is_uid: false }), - jmap: self.jmap, - imap: self.imap, + jmap, + imap: self.imap.imap_inner, instance: session.instance, state: State::NotAuthenticated { auth_failures: 0 }, span: session.span, @@ -68,10 +70,6 @@ impl SessionManager for ManageSieveSessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } - - fn is_ip_blocked(&self, addr: &std::net::IpAddr) -> bool { - self.jmap.directory.blocked_ips.is_blocked(addr) - } } impl Session { @@ -83,9 +81,9 @@ impl Session { tokio::select! { result = tokio::time::timeout( if !matches!(self.state, State::NotAuthenticated {..}) { - self.imap.timeout_auth + self.jmap.core.imap.timeout_auth } else { - self.imap.timeout_unauth + self.jmap.core.imap.timeout_unauth }, self.read(&mut buf)) => { match result { diff --git a/crates/managesieve/src/lib.rs b/crates/managesieve/src/lib.rs index 65c5f9f3..9c95268d 100644 --- a/crates/managesieve/src/lib.rs +++ b/crates/managesieve/src/lib.rs @@ -24,11 +24,7 @@ pub mod core; pub mod op; -static SERVER_GREETING: &str = concat!( - "Stalwart ManageSieve v", - env!("CARGO_PKG_VERSION"), - " at your service." -); +static SERVER_GREETING: &str = "Stalwart ManageSieve at your service."; #[cfg(test)] mod tests { diff --git a/crates/managesieve/src/op/authenticate.rs b/crates/managesieve/src/op/authenticate.rs index f5e8c2f4..8ff32a6f 100644 --- a/crates/managesieve/src/op/authenticate.rs +++ b/crates/managesieve/src/op/authenticate.rs @@ -21,17 +21,19 @@ * for more details. */ -use std::sync::Arc; - -use directory::AuthResult; +use common::{ + listener::{limiter::ConcurrencyLimiter, SessionStream}, + AuthResult, +}; use imap::op::authenticate::{decode_challenge_oauth, decode_challenge_plain}; use imap_proto::{ protocol::authenticate::Mechanism, receiver::{self, Request}, }; +use jmap::auth::rate_limit::ConcurrencyLimiters; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; -use utils::listener::SessionStream; +use std::sync::Arc; use crate::core::{Command, Session, State, StatusResponse}; @@ -131,34 +133,36 @@ impl Session { if let Some(access_token) = access_token { // Enforce concurrency limits - let in_flight = self - .imap + let in_flight = match self .get_concurrency_limiter(access_token.primary_id()) - .concurrent_requests - .is_allowed(); - if let Some(in_flight) = in_flight { - // Cache access token - let access_token = Arc::new(access_token); - self.jmap.cache_access_token(access_token.clone()); + .map(|limiter| limiter.concurrent_requests.is_allowed()) + { + Some(Some(limiter)) => Some(limiter), + None => None, + Some(None) => { + tracing::debug!(parent: &self.span, + event = "disconnect", + "Too many concurrent connection.", + ); + return Err(StatusResponse::bye("Too many concurrent connections.")); + } + }; - // Create session - self.state = State::Authenticated { - access_token, - in_flight, - }; + // Cache access token + let access_token = Arc::new(access_token); + self.jmap.cache_access_token(access_token.clone()); - self.handle_capability("Authentication successful").await - } else { - tracing::debug!(parent: &self.span, - event = "disconnect", - "Too many concurrent connection.", - ); - Err(StatusResponse::bye("Too many concurrent connections.")) - } + // Create session + self.state = State::Authenticated { + access_token, + in_flight, + }; + + self.handle_capability("Authentication successful").await } else { match &self.state { State::NotAuthenticated { auth_failures } - if *auth_failures < self.imap.max_auth_failures => + if *auth_failures < self.jmap.core.imap.max_auth_failures => { self.state = State::NotAuthenticated { auth_failures: auth_failures + 1, @@ -182,4 +186,21 @@ impl Session { Ok(StatusResponse::ok("Unauthenticate successful.").into_bytes()) } + + pub fn get_concurrency_limiter(&self, account_id: u32) -> Option> { + let rate = self.jmap.core.imap.rate_concurrent?; + self.imap + .rate_limiter + .get(&account_id) + .map(|limiter| limiter.clone()) + .unwrap_or_else(|| { + let limiter = Arc::new(ConcurrencyLimiters { + concurrent_requests: ConcurrencyLimiter::new(rate), + concurrent_uploads: ConcurrencyLimiter::new(rate), + }); + self.imap.rate_limiter.insert(account_id, limiter.clone()); + limiter + }) + .into() + } } diff --git a/crates/managesieve/src/op/capability.rs b/crates/managesieve/src/op/capability.rs index 8f1998d4..7495ac2b 100644 --- a/crates/managesieve/src/op/capability.rs +++ b/crates/managesieve/src/op/capability.rs @@ -21,39 +21,38 @@ * for more details. */ -use jmap::api::session::Capabilities; -use utils::listener::SessionStream; +use common::listener::SessionStream; +use jmap_proto::request::capability::Capabilities; use crate::core::{Session, StatusResponse}; impl Session { pub async fn handle_capability(&self, message: &'static str) -> super::OpResult { let mut response = Vec::with_capacity(128); - response.extend_from_slice(b"\"IMPLEMENTATION\" \"Stalwart ManageSieve v"); - response.extend_from_slice(env!("CARGO_PKG_VERSION").as_bytes()); - response.extend_from_slice(b"\"\r\n"); + response.extend_from_slice(b"\"IMPLEMENTATION\" \"Stalwart ManageSieve\"\r\n"); response.extend_from_slice(b"\"VERSION\" \"1.0\"\r\n"); if !self.stream.is_tls() { response.extend_from_slice(b"\"STARTTLS\"\r\n"); } - if self.stream.is_tls() || self.imap.allow_plain_auth { + if self.stream.is_tls() || self.jmap.core.imap.allow_plain_auth { response.extend_from_slice(b"\"SASL\" \"PLAIN OAUTHBEARER\"\r\n"); } else { response.extend_from_slice(b"\"SASL\" \"\"\r\n"); }; - if let Some(sieve) = self - .jmap - .config - .capabilities - .account - .iter() - .find_map(|(_, item)| { - if let Capabilities::SieveAccount(sieve) = item { - Some(sieve) - } else { - None - } - }) + if let Some(sieve) = + self.jmap + .core + .jmap + .capabilities + .account + .iter() + .find_map(|(_, item)| { + if let Capabilities::SieveAccount(sieve) = item { + Some(sieve) + } else { + None + } + }) { response.extend_from_slice(b"\"SIEVE\" \""); response.extend_from_slice(sieve.extensions.join(" ").as_bytes()); diff --git a/crates/managesieve/src/op/checkscript.rs b/crates/managesieve/src/op/checkscript.rs index 6158e4db..dd839e03 100644 --- a/crates/managesieve/src/op/checkscript.rs +++ b/crates/managesieve/src/op/checkscript.rs @@ -33,7 +33,9 @@ impl Session { } self.jmap - .sieve_compiler + .core + .sieve + .untrusted_compiler .compile(&request.tokens.into_iter().next().unwrap().unwrap_bytes()) .map(|_| StatusResponse::ok("Script is valid.").into_bytes()) .map_err(|err| StatusResponse::no(err.to_string())) diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 94aa6e8c..452ade76 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -67,7 +67,7 @@ impl Session { .await? .map(|ids| ids.len() as usize) .unwrap_or(0) - > self.jmap.config.sieve_max_scripts + > self.jmap.core.jmap.sieve_max_scripts { return Err( StatusResponse::no("Too many scripts.").with_code(ResponseCode::QuotaMaxScripts) @@ -75,7 +75,13 @@ impl Session { } // Compile script - match self.jmap.sieve_compiler.compile(&script_bytes) { + match self + .jmap + .core + .sieve + .untrusted_compiler + .compile(&script_bytes) + { Ok(compiled_script) => { script_bytes.extend(bincode::serialize(&compiled_script).unwrap_or_default()); } @@ -216,7 +222,7 @@ impl Session { ) -> Result, StatusResponse> { if name.is_empty() { Err(StatusResponse::no("Script name cannot be empty.")) - } else if name.len() > self.jmap.config.sieve_max_script_name { + } else if name.len() > self.jmap.core.jmap.sieve_max_script_name { Err(StatusResponse::no("Script name is too long.")) } else if name.eq_ignore_ascii_case("vacation") { Err(StatusResponse::no( diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 548f480a..15e8cd15 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -16,6 +16,7 @@ store = { path = "../store" } utils = { path = "../utils" } nlp = { path = "../nlp" } directory = { path = "../directory" } +common = { path = "../common" } mail-auth = { version = "0.3" } mail-send = { version = "0.4", default-features = false, features = ["cram-md5"] } mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } diff --git a/crates/smtp/src/config/auth.rs b/crates/smtp/src/config/auth.rs deleted file mode 100644 index 2fdfb96b..00000000 --- a/crates/smtp/src/config/auth.rs +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{sync::Arc, time::Duration}; - -use mail_auth::{ - common::crypto::{Algorithm, Ed25519Key, HashAlgorithm, RsaKey, Sha256, SigningKey}, - dkim::{Canonicalization, Done}, -}; -use mail_parser::decoders::base64::base64_decode; -use utils::{ - config::{ - if_block::IfBlock, - utils::{AsKey, ConstantValue, ParseValue}, - Config, - }, - expr::{self, Constant, Token}, -}; - -use crate::core::eval::*; - -use super::{ - map_expr_token, ArcAuthConfig, ArcSealer, ConfigContext, DkimAuthConfig, DkimCanonicalization, - DkimSigner, DmarcAuthConfig, IpRevAuthConfig, MailAuthConfig, SpfAuthConfig, VerifyStrategy, -}; - -pub trait ConfigAuth { - fn parse_mail_auth(&self) -> super::Result; - fn parse_signatures(&self, ctx: &mut ConfigContext) -> super::Result<()>; -} - -impl ConfigAuth for Config { - fn parse_mail_auth(&self) -> super::Result { - let fn_sender_keys = |name: &str| -> super::Result { - map_expr_token::( - name, - &[ - V_SENDER, - V_SENDER_DOMAIN, - V_PRIORITY, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - ], - ) - }; - let fn_conn_keys = |name: &str| -> super::Result { - map_expr_token::(name, &[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP]) - }; - - Ok(MailAuthConfig { - dkim: DkimAuthConfig { - verify: self - .parse_if_block("auth.dkim.verify", fn_sender_keys)? - .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), - sign: self - .parse_if_block("auth.dkim.sign", fn_sender_keys)? - .unwrap_or_default(), - }, - arc: ArcAuthConfig { - verify: self - .parse_if_block("auth.arc.verify", fn_sender_keys)? - .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), - seal: self - .parse_if_block("auth.arc.seal", fn_sender_keys)? - .unwrap_or_default(), - }, - spf: SpfAuthConfig { - verify_ehlo: self - .parse_if_block("auth.spf.verify.ehlo", fn_conn_keys)? - .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), - verify_mail_from: self - .parse_if_block("auth.spf.verify.mail-from", fn_conn_keys)? - .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), - }, - dmarc: DmarcAuthConfig { - verify: self - .parse_if_block("auth.dmarc.verify", fn_sender_keys)? - .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), - }, - iprev: IpRevAuthConfig { - verify: self - .parse_if_block("auth.iprev.verify", fn_conn_keys)? - .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), - }, - }) - } - - #[allow(clippy::type_complexity)] - fn parse_signatures(&self, ctx: &mut ConfigContext) -> super::Result<()> { - for id in self.sub_keys("signature", ".algorithm") { - let (signer, sealer) = - match self.property_require::(("signature", id, "algorithm"))? { - Algorithm::RsaSha256 => { - let pk = self.value_require(("signature", id, "private-key"))?; - let key = RsaKey::::from_rsa_pem(pk) - .or_else(|_| RsaKey::::from_pkcs8_pem(pk)) - .map_err(|err| { - format!( - "Failed to build RSA key for {}: {}", - ("signature", id, "private-key",).as_key(), - err - ) - })?; - let key_clone = RsaKey::::from_rsa_pem(pk) - .or_else(|_| RsaKey::::from_pkcs8_pem(pk)) - .map_err(|err| { - format!( - "Failed to build RSA key for {}: {}", - ("signature", id, "private-key",).as_key(), - err - ) - })?; - let (signer, sealer) = parse_signature(self, id, key_clone, key)?; - (DkimSigner::RsaSha256(signer), ArcSealer::RsaSha256(sealer)) - } - Algorithm::Ed25519Sha256 => { - let mut public_key = vec![]; - let mut private_key = vec![]; - - for (key, key_bytes) in [ - (("signature", id, "public-key"), &mut public_key), - (("signature", id, "private-key"), &mut private_key), - ] { - let mut contents = self.value_require(key)?.as_bytes().iter().copied(); - let mut base64 = vec![]; - - 'outer: while let Some(ch) = contents.next() { - if !ch.is_ascii_whitespace() { - if ch == b'-' { - for ch in contents.by_ref() { - if ch == b'\n' { - break; - } - } - } else { - base64.push(ch); - } - - for ch in contents.by_ref() { - if ch == b'-' { - break 'outer; - } else if !ch.is_ascii_whitespace() { - base64.push(ch); - } - } - } - } - - *key_bytes = base64_decode(&base64).ok_or_else(|| { - format!("Failed to base64 decode key for {}.", key.as_key(),) - })?; - } - - let key = Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key) - .or_else(|_| { - Ed25519Key::from_seed_and_public_key(&private_key, &public_key) - }) - .map_err(|err| { - format!("Failed to build ED25519 key for signature {id:?}: {err}") - })?; - let key_clone = Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key) - .or_else(|_| { - Ed25519Key::from_seed_and_public_key(&private_key, &public_key) - }) - .map_err(|err| { - format!("Failed to build ED25519 key for signature {id:?}: {err}") - })?; - - let (signer, sealer) = parse_signature(self, id, key_clone, key)?; - ( - DkimSigner::Ed25519Sha256(signer), - ArcSealer::Ed25519Sha256(sealer), - ) - } - Algorithm::RsaSha1 => { - return Err(format!( - "Could not build signature {id:?}: SHA1 signatures are deprecated.", - )) - } - }; - ctx.signers.insert(id.to_string(), Arc::new(signer)); - ctx.sealers.insert(id.to_string(), Arc::new(sealer)); - } - - Ok(()) - } -} - -fn parse_signature>( - config: &Config, - id: &str, - key_dkim: T, - key_arc: U, -) -> super::Result<( - mail_auth::dkim::DkimSigner, - mail_auth::arc::ArcSealer, -)> { - let domain = config.value_require(("signature", id, "domain"))?; - let selector = config.value_require(("signature", id, "selector"))?; - let mut headers = config - .values(("signature", id, "headers")) - .filter_map(|(_, v)| { - if !v.is_empty() { - v.to_string().into() - } else { - None - } - }) - .collect::>(); - if headers.is_empty() { - headers = vec![ - "From".to_string(), - "To".to_string(), - "Date".to_string(), - "Subject".to_string(), - "Message-ID".to_string(), - ]; - } - - let mut signer = mail_auth::dkim::DkimSigner::from_key(key_dkim) - .domain(domain) - .selector(selector) - .headers(headers.clone()); - if !headers - .iter() - .any(|h| h.eq_ignore_ascii_case("DKIM-Signature")) - { - headers.push("DKIM-Signature".to_string()); - } - let mut sealer = mail_auth::arc::ArcSealer::from_key(key_arc) - .domain(domain) - .selector(selector) - .headers(headers); - - if let Some(c) = - config.property::(("signature", id, "canonicalization"))? - { - signer = signer - .body_canonicalization(c.body) - .header_canonicalization(c.headers); - sealer = sealer - .body_canonicalization(c.body) - .header_canonicalization(c.headers); - } - - if let Some(c) = config.property::(("signature", id, "expire"))? { - signer = signer.expiration(c.as_secs()); - sealer = sealer.expiration(c.as_secs()); - } - - if let Some(true) = config.property::(("signature", id, "set-body-length"))? { - signer = signer.body_length(true); - sealer = sealer.body_length(true); - } - - if let Some(true) = config.property::(("signature", id, "report"))? { - signer = signer.reporting(true); - } - - if let Some(auid) = config.property::(("signature", id, "auid"))? { - signer = signer.agent_user_identifier(auid); - } - - if let Some(atps) = config.property::(("signature", id, "third-party"))? { - signer = signer.atps(atps); - } - - if let Some(atpsh) = config.property::(("signature", id, "third-party-algo"))? { - signer = signer.atpsh(atpsh); - } - - Ok((signer, sealer)) -} - -impl<'x> TryFrom> for VerifyStrategy { - type Error = (); - - fn try_from(value: expr::Variable<'x>) -> Result { - match value { - expr::Variable::Integer(c) => match c { - 2 => Ok(VerifyStrategy::Relaxed), - 3 => Ok(VerifyStrategy::Strict), - 4 => Ok(VerifyStrategy::Disable), - _ => Err(()), - }, - _ => Err(()), - } - } -} - -impl From for Constant { - fn from(value: VerifyStrategy) -> Self { - Constant::Integer(match value { - VerifyStrategy::Relaxed => 2, - VerifyStrategy::Strict => 3, - VerifyStrategy::Disable => 4, - }) - } -} - -impl ParseValue for VerifyStrategy { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - match value { - "relaxed" => Ok(VerifyStrategy::Relaxed), - "strict" => Ok(VerifyStrategy::Strict), - "disable" | "disabled" | "never" | "none" => Ok(VerifyStrategy::Disable), - _ => Err(format!( - "Invalid value {:?} for key {:?}.", - value, - key.as_key() - )), - } - } -} - -impl ConstantValue for VerifyStrategy {} - -impl ParseValue for DkimCanonicalization { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - if let Some((headers, body)) = value.split_once('/') { - Ok(DkimCanonicalization { - headers: Canonicalization::parse_value(key.clone(), headers.trim())?, - body: Canonicalization::parse_value(key, body.trim())?, - }) - } else { - let c = Canonicalization::parse_value(key, value)?; - Ok(DkimCanonicalization { - headers: c, - body: c, - }) - } - } -} - -impl Default for DkimCanonicalization { - fn default() -> Self { - Self { - headers: Canonicalization::Relaxed, - body: Canonicalization::Relaxed, - } - } -} diff --git a/crates/smtp/src/config/mod.rs b/crates/smtp/src/config/mod.rs deleted file mode 100644 index 4d79e6fe..00000000 --- a/crates/smtp/src/config/mod.rs +++ /dev/null @@ -1,457 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -pub mod auth; -pub mod queue; -pub mod report; -pub mod resolver; -pub mod scripts; -pub mod session; -pub mod shared; -pub mod throttle; - -use std::{net::SocketAddr, sync::Arc, time::Duration}; - -use ahash::AHashMap; -use directory::Directories; -use mail_auth::{ - common::crypto::{Ed25519Key, RsaKey, Sha256}, - dkim::{Canonicalization, Done}, -}; -use mail_send::Credentials; -use sieve::Sieve; -use store::Stores; -use utils::{ - config::{if_block::IfBlock, utils::ConstantValue, Rate, ServerProtocol}, - expr::{Expression, Token}, - snowflake::SnowflakeIdGenerator, -}; - -use crate::{ - core::eval::{FUNCTIONS_MAP, VARIABLES_MAP}, - inbound::milter, -}; - -#[derive(Debug, PartialEq, Eq, Clone)] -pub enum StringMatch { - Equal(String), - StartsWith(String), - EndsWith(String), -} - -#[derive(Debug, Default)] -#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] -pub struct Throttle { - pub expr: Expression, - pub keys: u16, - pub concurrency: Option, - pub rate: Option, -} - -pub const THROTTLE_RCPT: u16 = 1 << 0; -pub const THROTTLE_RCPT_DOMAIN: u16 = 1 << 1; -pub const THROTTLE_SENDER: u16 = 1 << 2; -pub const THROTTLE_SENDER_DOMAIN: u16 = 1 << 3; -pub const THROTTLE_AUTH_AS: u16 = 1 << 4; -pub const THROTTLE_LISTENER: u16 = 1 << 5; -pub const THROTTLE_MX: u16 = 1 << 6; -pub const THROTTLE_REMOTE_IP: u16 = 1 << 7; -pub const THROTTLE_LOCAL_IP: u16 = 1 << 8; -pub const THROTTLE_HELO_DOMAIN: u16 = 1 << 9; - -pub struct Connect { - pub script: IfBlock, -} - -pub struct Ehlo { - pub script: IfBlock, - pub require: IfBlock, - pub reject_non_fqdn: IfBlock, -} - -pub struct Extensions { - pub pipelining: IfBlock, - pub chunking: IfBlock, - pub requiretls: IfBlock, - pub dsn: IfBlock, - pub vrfy: IfBlock, - pub expn: IfBlock, - pub no_soliciting: IfBlock, - pub future_release: IfBlock, - pub deliver_by: IfBlock, - pub mt_priority: IfBlock, -} - -pub struct Auth { - pub directory: IfBlock, - pub mechanisms: IfBlock, - pub require: IfBlock, - pub allow_plain_text: IfBlock, - pub must_match_sender: IfBlock, - pub errors_max: IfBlock, - pub errors_wait: IfBlock, -} - -pub struct Mail { - pub script: IfBlock, - pub rewrite: IfBlock, -} - -pub struct Rcpt { - pub script: IfBlock, - pub relay: IfBlock, - pub directory: IfBlock, - pub rewrite: IfBlock, - - // Errors - pub errors_max: IfBlock, - pub errors_wait: IfBlock, - - // Limits - pub max_recipients: IfBlock, -} - -pub struct Data { - pub script: IfBlock, - pub pipe_commands: Vec, - pub milters: Vec, - - // Limits - pub max_messages: IfBlock, - pub max_message_size: IfBlock, - pub max_received_headers: IfBlock, - - // Headers - pub add_received: IfBlock, - pub add_received_spf: IfBlock, - pub add_return_path: IfBlock, - pub add_auth_results: IfBlock, - pub add_message_id: IfBlock, - pub add_date: IfBlock, -} - -pub struct Pipe { - pub command: IfBlock, - pub arguments: IfBlock, - pub timeout: IfBlock, -} - -pub struct Milter { - pub enable: IfBlock, - pub addrs: Vec, - pub hostname: String, - pub port: u16, - pub timeout_connect: Duration, - pub timeout_command: Duration, - pub timeout_data: Duration, - pub tls: bool, - pub tls_allow_invalid_certs: bool, - pub tempfail_on_error: bool, - pub max_frame_len: usize, - pub protocol_version: milter::Version, - pub flags_actions: Option, - pub flags_protocol: Option, -} - -pub struct SessionConfig { - pub timeout: IfBlock, - pub duration: IfBlock, - pub transfer_limit: IfBlock, - pub throttle: SessionThrottle, - - pub connect: Connect, - pub ehlo: Ehlo, - pub auth: Auth, - pub mail: Mail, - pub rcpt: Rcpt, - pub data: Data, - pub extensions: Extensions, -} - -pub struct SessionThrottle { - pub connect: Vec, - pub mail_from: Vec, - pub rcpt_to: Vec, -} - -pub struct RelayHost { - pub address: String, - pub port: u16, - pub protocol: ServerProtocol, - pub auth: Option>, - pub tls_implicit: bool, - pub tls_allow_invalid_certs: bool, -} - -pub struct QueueConfig { - // Schedule - pub retry: IfBlock, - pub notify: IfBlock, - pub expire: IfBlock, - - // Outbound - pub hostname: IfBlock, - pub next_hop: IfBlock, - pub max_mx: IfBlock, - pub max_multihomed: IfBlock, - pub ip_strategy: IfBlock, - pub source_ip: QueueOutboundSourceIp, - pub tls: QueueOutboundTls, - pub dsn: Dsn, - - // Timeouts - pub timeout: QueueOutboundTimeout, - - // Throttle and Quotas - pub throttle: QueueThrottle, - pub quota: QueueQuotas, -} - -pub struct QueueOutboundSourceIp { - pub ipv4: IfBlock, - pub ipv6: IfBlock, -} - -pub struct ReportConfig { - pub submitter: IfBlock, - pub analysis: ReportAnalysis, - - pub dkim: Report, - pub spf: Report, - pub dmarc: Report, - pub dmarc_aggregate: AggregateReport, - pub tls: AggregateReport, -} - -pub struct ReportAnalysis { - pub addresses: Vec, - pub forward: bool, - pub store: Option, - pub report_id: SnowflakeIdGenerator, -} - -pub enum AddressMatch { - StartsWith(String), - EndsWith(String), - Equals(String), -} - -pub struct Dsn { - pub name: IfBlock, - pub address: IfBlock, - pub sign: IfBlock, -} - -pub struct AggregateReport { - pub name: IfBlock, - pub address: IfBlock, - pub org_name: IfBlock, - pub contact_info: IfBlock, - pub send: IfBlock, - pub sign: IfBlock, - pub max_size: IfBlock, -} - -pub struct Report { - pub name: IfBlock, - pub address: IfBlock, - pub subject: IfBlock, - pub sign: IfBlock, - pub send: IfBlock, -} - -pub struct QueueOutboundTls { - pub dane: IfBlock, - pub mta_sts: IfBlock, - pub start: IfBlock, - pub invalid_certs: IfBlock, -} - -pub struct QueueOutboundTimeout { - pub connect: IfBlock, - pub greeting: IfBlock, - pub tls: IfBlock, - pub ehlo: IfBlock, - pub mail: IfBlock, - pub rcpt: IfBlock, - pub data: IfBlock, - pub mta_sts: IfBlock, -} - -#[derive(Debug)] -pub struct QueueThrottle { - pub sender: Vec, - pub rcpt: Vec, - pub host: Vec, -} - -pub struct QueueQuotas { - pub sender: Vec, - pub rcpt: Vec, - pub rcpt_domain: Vec, -} - -pub struct QueueQuota { - pub expr: Expression, - pub keys: u16, - pub size: Option, - pub messages: Option, -} - -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] -pub enum AggregateFrequency { - Hourly, - Daily, - Weekly, - #[default] - Never, -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct TlsStrategy { - pub dane: RequireOptional, - pub mta_sts: RequireOptional, - pub tls: RequireOptional, -} - -#[derive(Debug, Clone, Copy, Default)] -pub enum RequireOptional { - #[default] - Optional, - Require, - Disable, -} - -pub struct MailAuthConfig { - pub dkim: DkimAuthConfig, - pub arc: ArcAuthConfig, - pub spf: SpfAuthConfig, - pub dmarc: DmarcAuthConfig, - pub iprev: IpRevAuthConfig, -} - -pub enum DkimSigner { - RsaSha256(mail_auth::dkim::DkimSigner, Done>), - Ed25519Sha256(mail_auth::dkim::DkimSigner), -} - -pub enum ArcSealer { - RsaSha256(mail_auth::arc::ArcSealer, Done>), - Ed25519Sha256(mail_auth::arc::ArcSealer), -} - -pub struct DkimAuthConfig { - pub verify: IfBlock, - pub sign: IfBlock, -} - -pub struct ArcAuthConfig { - pub verify: IfBlock, - pub seal: IfBlock, -} - -pub struct SpfAuthConfig { - pub verify_ehlo: IfBlock, - pub verify_mail_from: IfBlock, -} -pub struct DmarcAuthConfig { - pub verify: IfBlock, -} - -pub struct IpRevAuthConfig { - pub verify: IfBlock, -} - -#[derive(Debug, Clone)] -pub struct DkimCanonicalization { - pub headers: Canonicalization, - pub body: Canonicalization, -} - -#[derive(Debug, Clone, Copy, Default)] -pub enum VerifyStrategy { - #[default] - Relaxed, - Strict, - Disable, -} - -#[derive(Default)] -pub struct ConfigContext { - pub directory: Directories, - pub stores: Stores, - pub scripts: AHashMap>, - pub signers: AHashMap>, - pub sealers: AHashMap>, -} - -impl ConfigContext { - pub fn new() -> Self { - Self { - ..Default::default() - } - } -} - -pub fn map_expr_token(name: &str, allowed_vars: &[u32]) -> Result { - VARIABLES_MAP - .iter() - .find(|(n, _)| n == &name) - .and_then(|(_, id)| { - if allowed_vars.contains(id) { - Some(Token::Variable(*id)) - } else { - None - } - }) - .or_else(|| { - FUNCTIONS_MAP - .iter() - .find(|(n, _, _)| n == &name) - .map(|(name, id, num_args)| Token::Function { - name: (*name).into(), - id: *id, - num_args: *num_args, - }) - }) - .or_else(|| { - F::parse_value("", name) - .map(|v| Token::Constant(v.into())) - .ok() - }) - .ok_or_else(|| format!("Invalid variable: {name:?}")) -} - -impl std::fmt::Debug for RelayHost { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RelayHost") - .field("address", &self.address) - .field("port", &self.port) - .field("protocol", &self.protocol) - .field("tls_implicit", &self.tls_implicit) - .field("tls_allow_invalid_certs", &self.tls_allow_invalid_certs) - .finish() - } -} - -pub type Result = std::result::Result; diff --git a/crates/smtp/src/config/queue.rs b/crates/smtp/src/config/queue.rs deleted file mode 100644 index 7b0f2711..00000000 --- a/crates/smtp/src/config/queue.rs +++ /dev/null @@ -1,396 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::time::Duration; - -use mail_auth::IpLookupStrategy; - -use crate::core::eval::*; - -use super::{ - map_expr_token, - throttle::{ConfigThrottle, ParseTrottleKey}, - Dsn, QueueConfig, QueueOutboundSourceIp, QueueOutboundTimeout, QueueOutboundTls, QueueQuota, - QueueQuotas, QueueThrottle, RequireOptional, THROTTLE_LOCAL_IP, THROTTLE_MX, THROTTLE_RCPT, - THROTTLE_RCPT_DOMAIN, THROTTLE_REMOTE_IP, THROTTLE_SENDER, THROTTLE_SENDER_DOMAIN, -}; -use utils::{ - config::{ - if_block::IfBlock, - utils::{AsKey, ConstantValue, NoConstants, ParseValue}, - Config, - }, - expr::{Constant, Expression, ExpressionItem, Variable}, -}; - -pub trait ConfigQueue { - fn parse_queue(&self) -> super::Result; - fn parse_queue_throttle(&self) -> super::Result; - fn parse_queue_quota(&self) -> super::Result; - fn parse_queue_quota_item(&self, prefix: impl AsKey) -> super::Result; -} - -impl ConfigQueue for Config { - fn parse_queue(&self) -> super::Result { - let rcpt_envelope_keys = &[V_RECIPIENT_DOMAIN, V_SENDER, V_SENDER_DOMAIN, V_PRIORITY]; - let sender_envelope_keys = &[V_SENDER, V_SENDER_DOMAIN, V_PRIORITY]; - let mx_envelope_keys = &[ - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_PRIORITY, - V_MX, - ]; - let host_envelope_keys = &[ - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_PRIORITY, - V_LOCAL_IP, - V_REMOTE_IP, - V_MX, - ]; - - let default_hostname = self.value_require("server.hostname")?; - - let config = QueueConfig { - retry: self - .parse_if_block("queue.schedule.retry", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), - notify: self - .parse_if_block("queue.schedule.notify", |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(86400))), - expire: self - .parse_if_block("queue.schedule.expire", |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 86400))), - hostname: self - .parse_if_block("queue.outbound.hostname", |name| { - map_expr_token::(name, sender_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(default_hostname.to_string())), - max_mx: self - .parse_if_block("queue.outbound.limits.mx", |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(5)), - max_multihomed: self - .parse_if_block("queue.outbound.limits.multihomed", |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(2)), - ip_strategy: self - .parse_if_block("queue.outbound.ip-strategy", |name| { - map_expr_token::(name, sender_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(IpLookupStrategy::Ipv4thenIpv6)), - source_ip: QueueOutboundSourceIp { - ipv4: self - .parse_if_block("queue.outbound.source-ip.v4", |name| { - map_expr_token::(name, mx_envelope_keys) - })? - .unwrap_or_default(), - ipv6: self - .parse_if_block("queue.outbound.source-ip.v6", |name| { - map_expr_token::(name, mx_envelope_keys) - })? - .unwrap_or_default(), - }, - next_hop: self - .parse_if_block("queue.outbound.next-hop", |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_default(), - tls: QueueOutboundTls { - dane: self - .parse_if_block("queue.outbound.tls.dane", |name| { - map_expr_token::(name, mx_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(RequireOptional::Optional)), - mta_sts: self - .parse_if_block("queue.outbound.tls.mta-sts", |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(RequireOptional::Optional)), - start: self - .parse_if_block("queue.outbound.tls.starttls", |name| { - map_expr_token::(name, mx_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(RequireOptional::Optional)), - invalid_certs: self - .parse_if_block("queue.outbound.tls.allow-invalid-certs", |name| { - map_expr_token::(name, mx_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(false)), - }, - throttle: self.parse_queue_throttle()?, - quota: self.parse_queue_quota()?, - timeout: QueueOutboundTimeout { - connect: self - .parse_if_block("queue.outbound.timeouts.connect", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), - greeting: self - .parse_if_block("queue.outbound.timeouts.greeting", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), - tls: self - .parse_if_block("queue.outbound.timeouts.tls", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(3 * 60))), - ehlo: self - .parse_if_block("queue.outbound.timeouts.ehlo", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), - mail: self - .parse_if_block("queue.outbound.timeouts.mail-from", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), - rcpt: self - .parse_if_block("queue.outbound.timeouts.rcpt-to", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), - data: self - .parse_if_block("queue.outbound.timeouts.data", |name| { - map_expr_token::(name, host_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(10 * 60))), - mta_sts: self - .parse_if_block("queue.outbound.timeouts.mta-sts", |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(10 * 60))), - }, - dsn: Dsn { - name: self - .parse_if_block("report.dsn.from-name", |name| { - map_expr_token::(name, sender_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new("Mail Delivery Subsystem".to_string())), - address: self - .parse_if_block("report.dsn.from-address", |name| { - map_expr_token::(name, sender_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(format!("MAILER-DAEMON@{default_hostname}"))), - sign: self - .parse_if_block("report.dsn.sign", |name| { - map_expr_token::(name, sender_envelope_keys) - })? - .unwrap_or_default(), - }, - }; - - Ok(config) - } - - fn parse_queue_throttle(&self) -> super::Result { - // Parse throttle - let mut throttle = QueueThrottle { - sender: Vec::new(), - rcpt: Vec::new(), - host: Vec::new(), - }; - let envelope_keys = [ - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_PRIORITY, - V_MX, - V_REMOTE_IP, - V_LOCAL_IP, - ]; - let all_throttles = self.parse_throttle( - "queue.throttle", - &envelope_keys, - THROTTLE_RCPT_DOMAIN - | THROTTLE_SENDER - | THROTTLE_SENDER_DOMAIN - | THROTTLE_MX - | THROTTLE_REMOTE_IP - | THROTTLE_LOCAL_IP, - )?; - for t in all_throttles { - if (t.keys & (THROTTLE_MX | THROTTLE_REMOTE_IP | THROTTLE_LOCAL_IP)) != 0 - || t.expr - .items() - .iter() - .any(|c| matches!(c, ExpressionItem::Variable(V_MX | V_REMOTE_IP | V_LOCAL_IP))) - { - throttle.host.push(t); - } else if (t.keys & (THROTTLE_RCPT_DOMAIN)) != 0 - || t.expr - .items() - .iter() - .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT_DOMAIN))) - { - throttle.rcpt.push(t); - } else { - throttle.sender.push(t); - } - } - - Ok(throttle) - } - - fn parse_queue_quota(&self) -> super::Result { - let mut capacities = QueueQuotas { - sender: Vec::new(), - rcpt: Vec::new(), - rcpt_domain: Vec::new(), - }; - - for array_pos in self.sub_keys("queue.quota", "") { - let quota = self.parse_queue_quota_item(("queue.quota", array_pos))?; - - if (quota.keys & THROTTLE_RCPT) != 0 - || quota - .expr - .items() - .iter() - .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT))) - { - capacities.rcpt.push(quota); - } else if (quota.keys & THROTTLE_RCPT_DOMAIN) != 0 - || quota - .expr - .items() - .iter() - .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT_DOMAIN))) - { - capacities.rcpt_domain.push(quota); - } else { - capacities.sender.push(quota); - } - } - - Ok(capacities) - } - - fn parse_queue_quota_item(&self, prefix: impl AsKey) -> super::Result { - let prefix = prefix.as_key(); - let mut keys = 0; - for (key_, value) in self.values((&prefix, "key")) { - let key = value.parse_throttle_key(key_)?; - if (key - & (THROTTLE_RCPT_DOMAIN | THROTTLE_RCPT | THROTTLE_SENDER | THROTTLE_SENDER_DOMAIN)) - != 0 - { - keys |= key; - } else { - return Err(format!( - "Key {value:?} is not available in this context for property {key_:?}" - )); - } - } - - let quota = QueueQuota { - expr: if let Some(expr) = self.value((&prefix, "match")) { - Expression::parse((&prefix, "match"), expr, |name| { - map_expr_token::( - name, - &[ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_PRIORITY, - ], - ) - })? - } else { - Expression::default() - }, - keys, - size: self - .property::((prefix.as_str(), "size"))? - .filter(|&v| v > 0), - messages: self - .property::((prefix.as_str(), "messages"))? - .filter(|&v| v > 0), - }; - - // Validate - if quota.size.is_none() && quota.messages.is_none() { - Err(format!( - concat!( - "Queue quota {:?} needs to define a ", - "valid 'size' and/or 'messages' property." - ), - prefix - )) - } else { - Ok(quota) - } - } -} - -impl ParseValue for RequireOptional { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - match value { - "optional" => Ok(RequireOptional::Optional), - "require" | "required" => Ok(RequireOptional::Require), - "disable" | "disabled" | "none" | "false" => Ok(RequireOptional::Disable), - _ => Err(format!( - "Invalid TLS option value {:?} for key {:?}.", - value, - key.as_key() - )), - } - } -} - -impl<'x> TryFrom> for RequireOptional { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - utils::expr::Variable::Integer(2) => Ok(RequireOptional::Optional), - utils::expr::Variable::Integer(1) => Ok(RequireOptional::Require), - utils::expr::Variable::Integer(0) => Ok(RequireOptional::Disable), - _ => Err(()), - } - } -} - -impl From for Constant { - fn from(value: RequireOptional) -> Self { - Constant::Integer(match value { - RequireOptional::Optional => 2, - RequireOptional::Require => 1, - RequireOptional::Disable => 0, - }) - } -} - -impl ConstantValue for RequireOptional {} diff --git a/crates/smtp/src/config/report.rs b/crates/smtp/src/config/report.rs deleted file mode 100644 index 930f5efa..00000000 --- a/crates/smtp/src/config/report.rs +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::time::Duration; - -use crate::core::eval::*; -use utils::{ - config::{ - if_block::IfBlock, - utils::{AsKey, ConstantValue, NoConstants, ParseValue}, - Config, - }, - expr::{Constant, Variable}, - snowflake::SnowflakeIdGenerator, -}; - -use super::{ - map_expr_token, AddressMatch, AggregateFrequency, AggregateReport, Report, ReportAnalysis, - ReportConfig, -}; - -pub trait ConfigReport { - fn parse_reports(&self) -> super::Result; - fn parse_report( - &self, - id: &str, - default_hostname: &str, - available_keys: &[u32], - ) -> super::Result; - fn parse_aggregate_report( - &self, - id: &str, - default_hostname: &str, - available_keys: &[u32], - ) -> super::Result; -} - -impl ConfigReport for Config { - fn parse_reports(&self) -> super::Result { - let sender_envelope_keys = &[ - V_SENDER, - V_SENDER_DOMAIN, - V_PRIORITY, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - ]; - let rcpt_envelope_keys = &[ - V_SENDER, - V_SENDER_DOMAIN, - V_PRIORITY, - V_REMOTE_IP, - V_LOCAL_IP, - V_RECIPIENT_DOMAIN, - ]; - let mut addresses = Vec::new(); - for address in self.properties::("report.analysis.addresses") { - addresses.push(address?.1); - } - - let default_hostname = self.value_require("server.hostname")?; - Ok(ReportConfig { - dkim: self.parse_report("dkim", default_hostname, sender_envelope_keys)?, - spf: self.parse_report("spf", default_hostname, sender_envelope_keys)?, - dmarc: self.parse_report("dmarc", default_hostname, sender_envelope_keys)?, - dmarc_aggregate: self.parse_aggregate_report( - "dmarc", - default_hostname, - sender_envelope_keys, - )?, - tls: self.parse_aggregate_report("tls", default_hostname, rcpt_envelope_keys)?, - submitter: self - .parse_if_block("report.submitter", |name| { - map_expr_token::(name, &[V_RECIPIENT_DOMAIN]) - })? - .unwrap_or_else(|| IfBlock::new(default_hostname.to_string())), - analysis: ReportAnalysis { - addresses, - forward: self.property("report.analysis.forward")?.unwrap_or(false), - store: self.property("report.analysis.store")?, - report_id: self - .property::("storage.cluster.node-id")? - .map(SnowflakeIdGenerator::with_node_id) - .unwrap_or_else(SnowflakeIdGenerator::new), - }, - }) - } - - fn parse_report( - &self, - id: &str, - default_hostname: &str, - available_keys: &[u32], - ) -> super::Result { - Ok(Report { - name: self - .parse_if_block(("report", id, "from-name"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new("Mail Delivery Subsystem".to_string())), - address: self - .parse_if_block(("report", id, "from-address"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(format!("MAILER-DAEMON@{default_hostname}"))), - subject: self - .parse_if_block(("report", id, "subject"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(format!("{} Report", id.to_ascii_uppercase()))), - sign: self - .parse_if_block(("report", id, "sign"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - send: self - .parse_if_block(("report", id, "send"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - }) - } - - fn parse_aggregate_report( - &self, - id: &str, - default_hostname: &str, - available_keys: &[u32], - ) -> super::Result { - let rcpt_envelope_keys = &[V_RECIPIENT_DOMAIN]; - - Ok(AggregateReport { - name: self - .parse_if_block(("report", id, "aggregate.from-name"), |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| { - IfBlock::new(format!("{} Aggregate Report", id.to_ascii_uppercase())) - }), - address: self - .parse_if_block(("report", id, "aggregate.from-address"), |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(format!("noreply-{id}@{default_hostname}"))), - org_name: self - .parse_if_block(("report", id, "aggregate.org-name"), |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_default(), - contact_info: self - .parse_if_block(("report", id, "aggregate.contact-info"), |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_default(), - send: self - .parse_if_block(("report", id, "aggregate.send"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(AggregateFrequency::Never)), - sign: self - .parse_if_block(("report", id, "aggregate.sign"), |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_default(), - max_size: self - .parse_if_block(("report", id, "aggregate.max-size"), |name| { - map_expr_token::(name, rcpt_envelope_keys) - })? - .unwrap_or_else(|| IfBlock::new(25 * 1024 * 1024)), - }) - } -} - -impl ParseValue for AggregateFrequency { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - match value { - "daily" | "day" => Ok(AggregateFrequency::Daily), - "hourly" | "hour" => Ok(AggregateFrequency::Hourly), - "weekly" | "week" => Ok(AggregateFrequency::Weekly), - "never" | "disable" | "false" => Ok(AggregateFrequency::Never), - _ => Err(format!( - "Invalid aggregate frequency value {:?} for key {:?}.", - value, - key.as_key() - )), - } - } -} - -impl From for Constant { - fn from(value: AggregateFrequency) -> Self { - match value { - AggregateFrequency::Never => 0.into(), - AggregateFrequency::Hourly => 2.into(), - AggregateFrequency::Daily => 3.into(), - AggregateFrequency::Weekly => 4.into(), - } - } -} - -impl<'x> TryFrom> for AggregateFrequency { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Integer(0) => Ok(AggregateFrequency::Never), - Variable::Integer(2) => Ok(AggregateFrequency::Hourly), - Variable::Integer(3) => Ok(AggregateFrequency::Daily), - Variable::Integer(4) => Ok(AggregateFrequency::Weekly), - _ => Err(()), - } - } -} - -impl ConstantValue for AggregateFrequency {} - -impl ParseValue for AddressMatch { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - if let Some(value) = value.strip_prefix('*').map(|v| v.trim()) { - if !value.is_empty() { - return Ok(AddressMatch::EndsWith(value.to_lowercase())); - } - } else if let Some(value) = value.strip_suffix('*').map(|v| v.trim()) { - if !value.is_empty() { - return Ok(AddressMatch::StartsWith(value.to_lowercase())); - } - } else if value.contains('@') { - return Ok(AddressMatch::Equals(value.trim().to_lowercase())); - } - Err(format!( - "Invalid address match value {:?} for key {:?}.", - value, - key.as_key() - )) - } -} diff --git a/crates/smtp/src/config/resolver.rs b/crates/smtp/src/config/resolver.rs deleted file mode 100644 index 2f1fb1da..00000000 --- a/crates/smtp/src/config/resolver.rs +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - io::Read, - net::{IpAddr, SocketAddr}, -}; - -use mail_auth::{ - common::lru::{DnsCache, LruCache}, - flate2::read::GzDecoder, - hickory_resolver::{ - config::{NameServerConfig, Protocol, ResolverConfig, ResolverOpts}, - system_conf::read_system_conf, - }, - Resolver, -}; - -use crate::{core::Resolvers, outbound::dane::DnssecResolver}; -use utils::{config::Config, suffixlist::PublicSuffix}; - -pub trait ConfigResolver { - fn build_resolvers(&self) -> super::Result; - fn parse_public_suffix(&self) -> super::Result; -} - -impl ConfigResolver for Config { - fn build_resolvers(&self) -> super::Result { - let (config, mut opts) = match self.value_require("resolver.type")? { - "cloudflare" => (ResolverConfig::cloudflare(), ResolverOpts::default()), - "cloudflare-tls" => (ResolverConfig::cloudflare_tls(), ResolverOpts::default()), - "quad9" => (ResolverConfig::quad9(), ResolverOpts::default()), - "quad9-tls" => (ResolverConfig::quad9_tls(), ResolverOpts::default()), - "google" => (ResolverConfig::google(), ResolverOpts::default()), - "system" => read_system_conf() - .map_err(|err| format!("Failed to read system DNS config: {err}"))?, - "custom" => { - let mut config = ResolverConfig::new(); - for (_, url) in self.values("resolver.custom") { - let (proto, host) = if let Some((proto, host)) = url.split_once("://") { - ( - match proto { - "udp" => Protocol::Udp, - "tcp" => Protocol::Tcp, - "tls" => Protocol::Tls, - _ => { - return Err(format!("Invalid custom resolver protocol {url:?}")) - } - }, - host, - ) - } else { - (Protocol::Udp, url) - }; - let (host, port) = if let Some((host, port)) = host.split_once(':') { - ( - host, - port.parse::().map_err(|err| { - format!("Invalid custom resolver port {port:?}: {err}") - })?, - ) - } else { - (host, 53) - }; - let host = host - .parse::() - .map_err(|err| format!("Invalid custom resolver IP {host:?}: {err}"))?; - config - .add_name_server(NameServerConfig::new(SocketAddr::new(host, port), proto)); - } - if !config.name_servers().is_empty() { - (config, ResolverOpts::default()) - } else { - return Err("At least one custom resolver must be specified.".to_string()); - } - } - other => return Err(format!("Unknown resolver type {other:?}.")), - }; - if let Some(concurrency) = self.property("resolver.concurrency")? { - opts.num_concurrent_reqs = concurrency; - } - if let Some(timeout) = self.property("resolver.timeout")? { - opts.timeout = timeout; - } - if let Some(preserve) = self.property("resolver.preserve-intermediates")? { - opts.preserve_intermediates = preserve; - } - if let Some(try_tcp_on_error) = self.property("resolver.try-tcp-on-error")? { - opts.try_tcp_on_error = try_tcp_on_error; - } - if let Some(attempts) = self.property("resolver.attempts")? { - opts.attempts = attempts; - } - - // Prepare DNSSEC resolver options - let config_dnssec = config.clone(); - let mut opts_dnssec = opts.clone(); - opts_dnssec.validate = true; - - let mut capacities = [1024usize; 5]; - for (pos, key) in ["txt", "mx", "ipv4", "ipv6", "ptr"].into_iter().enumerate() { - if let Some(capacity) = self.property(("cache.resolver", key))? { - capacities[pos] = capacity; - } - } - - Ok(Resolvers { - dns: Resolver::with_capacities( - config, - opts, - capacities[0], - capacities[1], - capacities[2], - capacities[3], - capacities[4], - ) - .map_err(|err| format!("Failed to build DNS resolver: {err}"))?, - dnssec: DnssecResolver::with_capacity(config_dnssec, opts_dnssec) - .map_err(|err| format!("Failed to build DNSSEC resolver: {err}"))?, - cache: crate::core::DnsCache { - tlsa: LruCache::with_capacity( - self.property("cache.resolver.tlsa.size")?.unwrap_or(1024), - ), - mta_sts: LruCache::with_capacity( - self.property("cache.resolver.mta-sts.size")? - .unwrap_or(1024), - ), - }, - }) - } - - fn parse_public_suffix(&self) -> super::Result { - let mut has_values = false; - for (_, value) in self.values("resolver.public-suffix") { - has_values = true; - let bytes = if value.starts_with("https://") || value.starts_with("http://") { - match tokio::task::block_in_place(|| { - reqwest::blocking::get(value).and_then(|r| { - if r.status().is_success() { - r.bytes().map(Ok) - } else { - Ok(Err(r)) - } - }) - }) { - Ok(Ok(bytes)) => bytes.to_vec(), - Ok(Err(response)) => { - tracing::warn!( - "Failed to fetch public suffixes from {value:?}: Status {status}", - value = value, - status = response.status() - ); - continue; - } - Err(err) => { - tracing::warn!( - "Failed to fetch public suffixes from {value:?}: {err}", - value = value, - err = err - ); - continue; - } - } - } else if let Some(filename) = value.strip_prefix("file://") { - match std::fs::read(filename) { - Ok(bytes) => bytes, - Err(err) => { - tracing::warn!( - "Failed to read public suffixes from {value:?}: {err}", - value = value, - err = err - ); - continue; - } - } - } else { - return Err(format!("Invalid public suffix file {value:?}")); - }; - let bytes = if value.ends_with(".gz") { - match GzDecoder::new(&bytes[..]) - .bytes() - .collect::, _>>() - { - Ok(bytes) => bytes, - Err(err) => { - tracing::warn!( - "Failed to decompress public suffixes from {value:?}: {err}", - value = value, - err = err - ); - continue; - } - } - } else { - bytes - }; - - match String::from_utf8(bytes) { - Ok(list) => { - return Ok(PublicSuffix::from(list.as_str())); - } - Err(err) => { - tracing::warn!( - "Failed to parse public suffixes from {value:?}: {err}", - value = value, - err = err - ); - } - } - } - - if has_values { - tracing::warn!("Failed to parse public suffixes from any source."); - } else { - tracing::warn!("No public suffixes list was specified."); - } - - Ok(PublicSuffix::default()) - } -} diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs deleted file mode 100644 index 847ceb5d..00000000 --- a/crates/smtp/src/config/scripts.rs +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - collections::HashSet, - time::{Duration, Instant}, -}; - -use ahash::AHashMap; -use nlp::bayes::cache::BayesTokenCache; -use parking_lot::RwLock; -use sieve::{compiler::grammar::Capability, Compiler, Runtime}; - -use crate::{ - core::SieveCore, - scripts::{functions::register_functions, plugins::RegisterSievePlugins}, -}; -use utils::{ - config::{utils::AsKey, Config}, - suffixlist::PublicSuffix, -}; - -use super::{resolver::ConfigResolver, ConfigContext}; - -pub trait ConfigSieve { - fn parse_sieve(&self, ctx: &mut ConfigContext) -> super::Result; -} - -#[derive(Default)] -pub struct SieveContext { - pub psl: PublicSuffix, - pub bayes_cache: BayesTokenCache, - pub remote_lists: RemoteLists, -} - -pub struct RemoteLists { - pub lists: RwLock>, -} - -pub struct RemoteList { - pub entries: HashSet, - pub expires: Instant, -} - -impl Default for RemoteLists { - fn default() -> Self { - Self { - lists: RwLock::new(AHashMap::new()), - } - } -} - -impl ConfigSieve for Config { - fn parse_sieve(&self, ctx: &mut ConfigContext) -> super::Result { - // Register functions - let mut fnc_map = register_functions().register_plugins(); - let sieve_ctx = SieveContext { - psl: self.parse_public_suffix()?, - bayes_cache: BayesTokenCache::new( - self.property_or_default("cache.bayes.capacity", "8192")?, - self.property_or_default("cache.bayes.ttl.positive", "1h")?, - self.property_or_default("cache.bayes.ttl.negative", "1h")?, - ), - remote_lists: Default::default(), - }; - - // Allocate compiler and runtime - let compiler = Compiler::new() - .with_max_string_size(52428800) - .with_max_variable_name_size(100) - .with_max_nested_blocks(50) - .with_max_nested_tests(50) - .with_max_nested_foreverypart(10) - .with_max_local_variables(8192) - .with_max_header_size(10240) - .with_max_includes(10) - .with_no_capability_check( - self.property_or_default("sieve.trusted.no-capability-check", "false")?, - ) - .register_functions(&mut fnc_map); - - let mut runtime = Runtime::new_with_context(sieve_ctx) - .without_capabilities([ - Capability::FileInto, - Capability::Vacation, - Capability::VacationSeconds, - Capability::Fcc, - Capability::Mailbox, - Capability::MailboxId, - Capability::MboxMetadata, - Capability::ServerMetadata, - Capability::ImapSieve, - Capability::Duplicate, - ]) - .with_capability(Capability::Expressions) - .with_capability(Capability::While) - .with_max_variable_size( - self.property_or_default("sieve.trusted.limits.variable-size", "52428800")?, - ) - .with_max_header_size(10240) - .with_valid_notification_uri("mailto") - .with_valid_ext_lists(ctx.stores.lookup_stores.keys().map(|k| k.to_string())) - .with_functions(&mut fnc_map); - - if let Some(value) = self.property("sieve.trusted.limits.redirects")? { - runtime.set_max_redirects(value); - } - if let Some(value) = self.property("sieve.trusted.limits.out-messages")? { - runtime.set_max_out_messages(value); - } - if let Some(value) = self.property("sieve.trusted.limits.cpu")? { - runtime.set_cpu_limit(value); - } - if let Some(value) = self.property("sieve.trusted.limits.nested-includes")? { - runtime.set_max_nested_includes(value); - } - if let Some(value) = self.property("sieve.trusted.limits.received-headers")? { - runtime.set_max_received_headers(value); - } - if let Some(value) = self.property::("sieve.trusted.limits.duplicate-expiry")? { - runtime.set_default_duplicate_expiry(value.as_secs()); - } - let hostname = if let Some(hostname) = self.value("sieve.trusted.hostname") { - hostname - } else { - self.value_require("server.hostname")? - }; - runtime.set_local_hostname(hostname.to_string()); - - // Parse scripts - for id in self.sub_keys("sieve.trusted.scripts", "") { - let key = ("sieve.trusted.scripts", id); - - let script = if !self.contains_key(key) { - let mut script = String::new(); - for sub_key in self.sub_keys(key, "") { - script.push_str(self.value_require(("sieve.trusted.scripts", id, sub_key))?); - } - script - } else { - self.value_require(key)?.to_string() - }; - - ctx.scripts.insert( - id.to_string(), - compiler - .compile(script.as_bytes()) - .map_err(|err| format!("Failed to compile Sieve script {id:?}: {err}"))? - .into(), - ); - } - - // Parse DKIM signatures - let mut sign = Vec::new(); - for (pos, id) in self.values("sieve.trusted.sign") { - if let Some(dkim) = ctx.signers.get(id) { - sign.push(dkim.clone()); - } else { - return Err(format!( - "No DKIM signer found with id {:?} for key {:?}.", - id, - ("sieve.trusted.sign", pos).as_key() - )); - } - } - - Ok(SieveCore { - runtime, - from_addr: self - .value("sieve.trusted.from-addr") - .map(|a| a.to_string()) - .unwrap_or(format!("MAILER-DAEMON@{hostname}")), - from_name: self - .value("sieve.trusted.from-name") - .unwrap_or("Mailer Daemon") - .to_string(), - return_path: self - .value("sieve.trusted.return-path") - .unwrap_or_default() - .to_string(), - sign, - }) - } -} diff --git a/crates/smtp/src/config/session.rs b/crates/smtp/src/config/session.rs deleted file mode 100644 index 14582b43..00000000 --- a/crates/smtp/src/config/session.rs +++ /dev/null @@ -1,633 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{net::ToSocketAddrs, time::Duration}; - -use smtp_proto::*; - -use crate::inbound::milter; - -use crate::core::eval::*; - -use super::{ - map_expr_token, throttle::ConfigThrottle, Auth, Connect, Data, Ehlo, Extensions, Mail, Milter, - Pipe, Rcpt, SessionConfig, SessionThrottle, THROTTLE_AUTH_AS, THROTTLE_HELO_DOMAIN, - THROTTLE_LISTENER, THROTTLE_LOCAL_IP, THROTTLE_RCPT, THROTTLE_RCPT_DOMAIN, THROTTLE_REMOTE_IP, - THROTTLE_SENDER, THROTTLE_SENDER_DOMAIN, -}; -use utils::{ - config::{ - if_block::IfBlock, - utils::{AsKey, ConstantValue, NoConstants, ParseValue}, - Config, - }, - expr::{Constant, ExpressionItem, Variable}, -}; - -pub trait ConfigSession { - fn parse_session_config(&self) -> super::Result; - fn parse_session_throttle(&self) -> super::Result; - fn parse_session_connect(&self) -> super::Result; - fn parse_extensions(&self) -> super::Result; - fn parse_session_ehlo(&self) -> super::Result; - fn parse_session_auth(&self) -> super::Result; - fn parse_session_mail(&self) -> super::Result; - fn parse_session_rcpt(&self) -> super::Result; - fn parse_session_data(&self) -> super::Result; - fn parse_pipes(&self, available_keys: &[u32]) -> super::Result>; - fn parse_milters(&self, available_keys: &[u32]) -> super::Result>; -} - -impl ConfigSession for Config { - fn parse_session_config(&self) -> super::Result { - let available_keys = &[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP]; - - Ok(SessionConfig { - duration: self - .parse_if_block("session.duration", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(15 * 60))), - transfer_limit: self - .parse_if_block("session.transfer-limit", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(250 * 1024 * 1024)), - timeout: self - .parse_if_block("session.timeout", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), - throttle: self.parse_session_throttle()?, - connect: self.parse_session_connect()?, - ehlo: self.parse_session_ehlo()?, - auth: self.parse_session_auth()?, - mail: self.parse_session_mail()?, - rcpt: self.parse_session_rcpt()?, - data: self.parse_session_data()?, - extensions: self.parse_extensions()?, - }) - } - - fn parse_session_throttle(&self) -> super::Result { - // Parse throttle - let mut throttle = SessionThrottle { - connect: Vec::new(), - mail_from: Vec::new(), - rcpt_to: Vec::new(), - }; - let all_throttles = self.parse_throttle( - "session.throttle", - &[ - V_SENDER, - V_SENDER_DOMAIN, - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - V_HELO_DOMAIN, - ], - THROTTLE_LISTENER - | THROTTLE_REMOTE_IP - | THROTTLE_LOCAL_IP - | THROTTLE_AUTH_AS - | THROTTLE_HELO_DOMAIN - | THROTTLE_RCPT - | THROTTLE_RCPT_DOMAIN - | THROTTLE_SENDER - | THROTTLE_SENDER_DOMAIN, - )?; - for t in all_throttles { - if (t.keys & (THROTTLE_RCPT | THROTTLE_RCPT_DOMAIN)) != 0 - || t.expr.items().iter().any(|c| { - matches!( - c, - ExpressionItem::Variable(V_RECIPIENT | V_RECIPIENT_DOMAIN) - ) - }) - { - throttle.rcpt_to.push(t); - } else if (t.keys - & (THROTTLE_SENDER - | THROTTLE_SENDER_DOMAIN - | THROTTLE_HELO_DOMAIN - | THROTTLE_AUTH_AS)) - != 0 - || t.expr.items().iter().any(|c| { - matches!( - c, - ExpressionItem::Variable( - V_SENDER | V_SENDER_DOMAIN | V_HELO_DOMAIN | V_AUTHENTICATED_AS - ) - ) - }) - { - throttle.mail_from.push(t); - } else { - throttle.connect.push(t); - } - } - - Ok(throttle) - } - - fn parse_session_connect(&self) -> super::Result { - let available_keys = &[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP]; - Ok(Connect { - script: self - .parse_if_block("session.connect.script", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - }) - } - - fn parse_extensions(&self) -> super::Result { - let available_keys = &[ - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_SENDER, - V_SENDER_DOMAIN, - V_AUTHENTICATED_AS, - ]; - - Ok(Extensions { - pipelining: self - .parse_if_block("session.extensions.pipelining", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - dsn: self - .parse_if_block("session.extensions.dsn", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - vrfy: self - .parse_if_block("session.extensions.vrfy", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - expn: self - .parse_if_block("session.extensions.expn", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - chunking: self - .parse_if_block("session.extensions.chunking", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - requiretls: self - .parse_if_block("session.extensions.requiretls", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - no_soliciting: self - .parse_if_block("session.extensions.no-soliciting", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(false)), - future_release: self - .parse_if_block("session.extensions.future-release", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - deliver_by: self - .parse_if_block("session.extensions.deliver-by", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - mt_priority: self - .parse_if_block("session.extensions.mt-priority", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - }) - } - - fn parse_session_ehlo(&self) -> super::Result { - let available_keys = &[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP]; - - Ok(Ehlo { - script: self - .parse_if_block("session.ehlo.script", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - require: self - .parse_if_block("session.ehlo.require", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - reject_non_fqdn: self - .parse_if_block("session.ehlo.reject-non-fqdn", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - }) - } - - fn parse_session_auth(&self) -> super::Result { - let available_keys = &[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP, V_HELO_DOMAIN]; - - Ok(Auth { - directory: self - .parse_if_block("session.auth.directory", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - mechanisms: self - .parse_if_block("session.auth.mechanisms", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - require: self - .parse_if_block("session.auth.require", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(false)), - errors_max: self - .parse_if_block("session.auth.errors.max", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(3)), - errors_wait: self - .parse_if_block("session.auth.errors.wait", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(30))), - allow_plain_text: self - .parse_if_block("session.auth.allow-plain-text", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(false)), - must_match_sender: self - .parse_if_block("session.auth.must-match-sender", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - }) - } - - fn parse_session_mail(&self) -> super::Result { - let available_keys = &[ - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_HELO_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - ]; - Ok(Mail { - script: self - .parse_if_block("session.mail.script", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - rewrite: self - .parse_if_block("session.mail.rewrite", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - }) - } - - fn parse_session_rcpt(&self) -> super::Result { - let available_keys = &[ - V_SENDER, - V_SENDER_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_HELO_DOMAIN, - ]; - let available_keys_full = &[ - V_SENDER, - V_SENDER_DOMAIN, - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_HELO_DOMAIN, - ]; - Ok(Rcpt { - script: self - .parse_if_block("session.rcpt.script", |name| { - map_expr_token::(name, available_keys_full) - })? - .unwrap_or_default(), - relay: self - .parse_if_block("session.rcpt.relay", |name| { - map_expr_token::(name, available_keys_full) - })? - .unwrap_or_else(|| IfBlock::new(false)), - directory: self - .parse_if_block("session.rcpt.directory", |name| { - map_expr_token::(name, available_keys_full) - })? - .unwrap_or_default(), - errors_max: self - .parse_if_block("session.rcpt.errors.max", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(10)), - errors_wait: self - .parse_if_block("session.rcpt.errors.wait", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(30))), - max_recipients: self - .parse_if_block("session.rcpt.max-recipients", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(100)), - rewrite: self - .parse_if_block("session.rcpt.rewrite", |name| { - map_expr_token::(name, available_keys_full) - })? - .unwrap_or_default(), - }) - } - - fn parse_session_data(&self) -> super::Result { - let available_keys = &[ - V_SENDER, - V_SENDER_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - V_HELO_DOMAIN, - ]; - Ok(Data { - script: self - .parse_if_block("session.data.script", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - max_messages: self - .parse_if_block("session.data.limits.messages", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(10)), - max_message_size: self - .parse_if_block("session.data.limits.size", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(25 * 1024 * 1024)), - max_received_headers: self - .parse_if_block("session.data.limits.received-headers", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(50)), - add_received: self - .parse_if_block("session.data.add-headers.received", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - add_received_spf: self - .parse_if_block("session.data.add-headers.received-spf", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - add_return_path: self - .parse_if_block("session.data.add-headers.return-path", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - add_auth_results: self - .parse_if_block("session.data.add-headers.auth-results", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - add_message_id: self - .parse_if_block("session.data.add-headers.message-id", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - add_date: self - .parse_if_block("session.data.add-headers.date", |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(true)), - pipe_commands: self.parse_pipes(available_keys)?, - milters: self.parse_milters(available_keys)?, - }) - } - - fn parse_pipes(&self, available_keys: &[u32]) -> super::Result> { - let mut pipes = Vec::new(); - for id in self.sub_keys("session.data.pipe", "") { - pipes.push(Pipe { - command: self - .parse_if_block(("session.data.pipe", id, "command"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - arguments: self - .parse_if_block(("session.data.pipe", id, "arguments"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - timeout: self - .parse_if_block(("session.data.pipe", id, "timeout"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_else(|| IfBlock::new(Duration::from_secs(30))), - }) - } - Ok(pipes) - } - - fn parse_milters(&self, available_keys: &[u32]) -> super::Result> { - let mut milters = Vec::new(); - for id in self.sub_keys("session.data.milter", "") { - let hostname = self - .value_require(("session.data.milter", id, "hostname"))? - .to_string(); - let port = self.property_require(("session.data.milter", id, "port"))?; - milters.push(Milter { - enable: self - .parse_if_block(("session.data.milter", id, "enable"), |name| { - map_expr_token::(name, available_keys) - })? - .unwrap_or_default(), - addrs: format!("{}:{}", hostname, port) - .to_socket_addrs() - .map_err(|err| format!("Unable to resolve milter hostname {hostname}: {err}"))? - .collect(), - hostname, - port, - timeout_connect: self - .property_or_default(("session.data.milter", id, "timeout.connect"), "30s")?, - timeout_command: self - .property_or_default(("session.data.milter", id, "timeout.command"), "30s")?, - timeout_data: self - .property_or_default(("session.data.milter", id, "timeout.data"), "60s")?, - tls: self.property_or_default(("session.data.milter", id, "tls"), "false")?, - tls_allow_invalid_certs: self.property_or_default( - ("session.data.milter", id, "allow-invalid-certs"), - "false", - )?, - tempfail_on_error: self.property_or_default( - ("session.data.milter", id, "options.tempfail-on-error"), - "true", - )?, - max_frame_len: self.property_or_default( - ("session.data.milter", id, "options.max-response-size"), - "52428800", - )?, - protocol_version: match self.property_or_default::( - ("session.data.milter", id, "options.version"), - "6", - )? { - 6 => milter::Version::V6, - 2 => milter::Version::V2, - v => return Err(format!("Unsupported milter protocol version: {}", v)), - }, - flags_actions: self.property(( - "session.data.milter", - id, - "options.flags.actions", - ))?, - flags_protocol: self.property(( - "session.data.milter", - id, - "options.flags.protocol", - ))?, - }) - } - Ok(milters) - } -} - -#[derive(Default)] -pub struct Mechanism(u64); - -impl ParseValue for Mechanism { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - Ok(Mechanism(match value.to_ascii_uppercase().as_str() { - "LOGIN" => AUTH_LOGIN, - "PLAIN" => AUTH_PLAIN, - "XOAUTH2" => AUTH_XOAUTH2, - "OAUTHBEARER" => AUTH_OAUTHBEARER, - /*"SCRAM-SHA-256-PLUS" => AUTH_SCRAM_SHA_256_PLUS, - "SCRAM-SHA-256" => AUTH_SCRAM_SHA_256, - "SCRAM-SHA-1-PLUS" => AUTH_SCRAM_SHA_1_PLUS, - "SCRAM-SHA-1" => AUTH_SCRAM_SHA_1, - "XOAUTH" => AUTH_XOAUTH, - "9798-M-DSA-SHA1" => AUTH_9798_M_DSA_SHA1, - "9798-M-ECDSA-SHA1" => AUTH_9798_M_ECDSA_SHA1, - "9798-M-RSA-SHA1-ENC" => AUTH_9798_M_RSA_SHA1_ENC, - "9798-U-DSA-SHA1" => AUTH_9798_U_DSA_SHA1, - "9798-U-ECDSA-SHA1" => AUTH_9798_U_ECDSA_SHA1, - "9798-U-RSA-SHA1-ENC" => AUTH_9798_U_RSA_SHA1_ENC, - "EAP-AES128" => AUTH_EAP_AES128, - "EAP-AES128-PLUS" => AUTH_EAP_AES128_PLUS, - "ECDH-X25519-CHALLENGE" => AUTH_ECDH_X25519_CHALLENGE, - "ECDSA-NIST256P-CHALLENGE" => AUTH_ECDSA_NIST256P_CHALLENGE, - "EXTERNAL" => AUTH_EXTERNAL, - "GS2-KRB5" => AUTH_GS2_KRB5, - "GS2-KRB5-PLUS" => AUTH_GS2_KRB5_PLUS, - "GSS-SPNEGO" => AUTH_GSS_SPNEGO, - "GSSAPI" => AUTH_GSSAPI, - "KERBEROS_V4" => AUTH_KERBEROS_V4, - "KERBEROS_V5" => AUTH_KERBEROS_V5, - "NMAS-SAMBA-AUTH" => AUTH_NMAS_SAMBA_AUTH, - "NMAS_AUTHEN" => AUTH_NMAS_AUTHEN, - "NMAS_LOGIN" => AUTH_NMAS_LOGIN, - "NTLM" => AUTH_NTLM, - "OAUTH10A" => AUTH_OAUTH10A, - "OPENID20" => AUTH_OPENID20, - "OTP" => AUTH_OTP, - "SAML20" => AUTH_SAML20, - "SECURID" => AUTH_SECURID, - "SKEY" => AUTH_SKEY, - "SPNEGO" => AUTH_SPNEGO, - "SPNEGO-PLUS" => AUTH_SPNEGO_PLUS, - "SXOVER-PLUS" => AUTH_SXOVER_PLUS, - "CRAM-MD5" => AUTH_CRAM_MD5, - "DIGEST-MD5" => AUTH_DIGEST_MD5, - "ANONYMOUS" => AUTH_ANONYMOUS,*/ - _ => { - return Err(format!( - "Unsupported mechanism {:?} for property {:?}.", - value, - key.as_key() - )) - } - })) - } -} - -impl<'x> TryFrom> for Mechanism { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Integer(value) => Ok(Mechanism(value as u64)), - Variable::Array(items) => { - let mut mechanism = 0; - - for item in items { - match item { - Variable::Integer(value) => mechanism |= value as u64, - _ => return Err(()), - } - } - - Ok(Mechanism(mechanism)) - } - _ => Err(()), - } - } -} - -impl From for Constant { - fn from(value: Mechanism) -> Self { - Constant::Integer(value.0 as i64) - } -} - -impl ConstantValue for Mechanism {} - -impl From for u64 { - fn from(value: Mechanism) -> Self { - value.0 - } -} - -impl From for Mechanism { - fn from(value: u64) -> Self { - Mechanism(value) - } -} diff --git a/crates/smtp/src/config/shared.rs b/crates/smtp/src/config/shared.rs deleted file mode 100644 index 81b4ffa5..00000000 --- a/crates/smtp/src/config/shared.rs +++ /dev/null @@ -1,109 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use ahash::AHashMap; -use mail_send::Credentials; -use utils::config::Config; - -use crate::core::Shared; - -use super::{ConfigContext, RelayHost}; - -pub trait ConfigShared { - fn parse_shared(&self, ctx: &ConfigContext) -> super::Result; - fn parse_host(&self, id: &str) -> super::Result; -} - -impl ConfigShared for Config { - fn parse_shared(&self, ctx: &ConfigContext) -> super::Result { - let mut relay_hosts = AHashMap::new(); - - for id in self.sub_keys("remote", ".address") { - relay_hosts.insert(id.to_string(), self.parse_host(id)?); - } - - Ok(Shared { - scripts: ctx.scripts.clone(), - signers: ctx.signers.clone(), - sealers: ctx.sealers.clone(), - directories: ctx.directory.directories.clone(), - lookup_stores: ctx.stores.lookup_stores.clone(), - relay_hosts, - default_directory: ctx - .directory - .directories - .get(self.value_require("storage.directory")?) - .ok_or_else(|| { - format!( - "Directory {:?} not found for key \"storage.directory\".", - self.value_require("storage.directory").unwrap() - ) - })? - .clone(), - default_data_store: ctx.stores.get_store(self, "storage.data")?, - default_lookup_store: self - .value_or_else("storage.lookup", "storage.data") - .and_then(|id| ctx.stores.lookup_stores.get(id)) - .ok_or_else(|| { - format!( - "Lookup store {:?} not found for key \"storage.lookup\".", - self.value_or_else("storage.lookup", "storage.data") - .unwrap() - ) - })? - .clone(), - default_blob_store: self - .value_or_else("storage.blob", "storage.data") - .and_then(|id| ctx.stores.blob_stores.get(id)) - .ok_or_else(|| { - format!( - "Lookup store {:?} not found for key \"storage.blob\".", - self.value_or_else("storage.blob", "storage.data").unwrap() - ) - })? - .clone(), - }) - } - - fn parse_host(&self, id: &str) -> super::Result { - let username = self.value(("remote", id, "auth.username")); - let secret = self.value(("remote", id, "auth.secret")); - - Ok(RelayHost { - address: self.property_require(("remote", id, "address"))?, - port: self.property_require(("remote", id, "port"))?, - protocol: self.property_require(("remote", id, "protocol"))?, - auth: if let (Some(username), Some(secret)) = (username, secret) { - Credentials::new(username.to_string(), secret.to_string()).into() - } else { - None - }, - tls_implicit: self - .property(("remote", id, "tls.implicit"))? - .unwrap_or(true), - tls_allow_invalid_certs: self - .property(("remote", id, "tls.allow-invalid-certs"))? - .unwrap_or(false), - }) - } -} diff --git a/crates/smtp/src/config/throttle.rs b/crates/smtp/src/config/throttle.rs deleted file mode 100644 index 0942db62..00000000 --- a/crates/smtp/src/config/throttle.rs +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use super::*; -use utils::config::{ - utils::{AsKey, NoConstants}, - Config, -}; - -pub trait ConfigThrottle { - fn parse_throttle( - &self, - prefix: impl AsKey, - available_envelope_keys: &[u32], - available_throttle_keys: u16, - ) -> super::Result>; - - fn parse_throttle_item( - &self, - prefix: impl AsKey, - available_envelope_keys: &[u32], - available_throttle_keys: u16, - ) -> super::Result; -} - -impl ConfigThrottle for Config { - fn parse_throttle( - &self, - prefix: impl AsKey, - available_envelope_keys: &[u32], - available_throttle_keys: u16, - ) -> super::Result> { - let prefix_ = prefix.as_key(); - let mut throttles = Vec::new(); - for array_pos in self.sub_keys(prefix, "") { - throttles.push(self.parse_throttle_item( - (&prefix_, array_pos), - available_envelope_keys, - available_throttle_keys, - )?); - } - - Ok(throttles) - } - - fn parse_throttle_item( - &self, - prefix: impl AsKey, - available_envelope_keys: &[u32], - available_throttle_keys: u16, - ) -> super::Result { - let prefix = prefix.as_key(); - let mut keys = 0; - for (key_, value) in self.values((&prefix, "key")) { - let key = value.parse_throttle_key(key_)?; - if (key & available_throttle_keys) != 0 { - keys |= key; - } else { - return Err(format!( - "Throttle key {value:?} is not available in this context for property {key_:?}" - )); - } - } - - let throttle = Throttle { - expr: if let Some(expr) = self.value((&prefix, "match")) { - Expression::parse((&prefix, "match"), expr, |name| { - map_expr_token::(name, available_envelope_keys) - })? - } else { - Expression::default() - }, - keys, - concurrency: self - .property::((prefix.as_str(), "concurrency"))? - .filter(|&v| v > 0), - rate: self - .property::((prefix.as_str(), "rate"))? - .filter(|v| v.requests > 0), - }; - - // Validate - if throttle.rate.is_none() && throttle.concurrency.is_none() { - Err(format!( - concat!( - "Throttle {:?} needs to define a ", - "valid 'rate' and/or 'concurrency' property." - ), - prefix - )) - } else { - Ok(throttle) - } - } -} - -pub trait ParseTrottleKey { - fn parse_throttle_key(&self, key: &str) -> super::Result; -} - -impl ParseTrottleKey for &str { - fn parse_throttle_key(&self, key: &str) -> super::Result { - match *self { - "rcpt" => Ok(THROTTLE_RCPT), - "rcpt_domain" => Ok(THROTTLE_RCPT_DOMAIN), - "sender" => Ok(THROTTLE_SENDER), - "sender_domain" => Ok(THROTTLE_SENDER_DOMAIN), - "authenticated_as" => Ok(THROTTLE_AUTH_AS), - "listener" => Ok(THROTTLE_LISTENER), - "mx" => Ok(THROTTLE_MX), - "remote_ip" => Ok(THROTTLE_REMOTE_IP), - "local_ip" => Ok(THROTTLE_LOCAL_IP), - "helo_domain" => Ok(THROTTLE_HELO_DOMAIN), - _ => Err(format!("Invalid throttle key {self:?} found in {key:?}")), - } - } -} diff --git a/crates/smtp/src/core/eval.rs b/crates/smtp/src/core/eval.rs deleted file mode 100644 index 485f1158..00000000 --- a/crates/smtp/src/core/eval.rs +++ /dev/null @@ -1,580 +0,0 @@ -use std::{borrow::Cow, cmp::Ordering, net::IpAddr, sync::Arc, vec::IntoIter}; - -use directory::Directory; -use mail_auth::IpLookupStrategy; -use sieve::Sieve; -use smtp_proto::IntoString; -use store::{Deserialize, LookupStore, Rows, Value}; -use utils::{ - config::if_block::IfBlock, - expr::{Expression, Variable}, -}; - -use crate::config::{ArcSealer, DkimSigner, RelayHost}; - -use super::{ResolveVariable, SMTP}; - -pub const V_RECIPIENT: u32 = 0; -pub const V_RECIPIENT_DOMAIN: u32 = 1; -pub const V_SENDER: u32 = 2; -pub const V_SENDER_DOMAIN: u32 = 3; -pub const V_MX: u32 = 4; -pub const V_HELO_DOMAIN: u32 = 5; -pub const V_AUTHENTICATED_AS: u32 = 6; -pub const V_LISTENER: u32 = 7; -pub const V_REMOTE_IP: u32 = 8; -pub const V_LOCAL_IP: u32 = 9; -pub const V_PRIORITY: u32 = 10; - -pub const F_IS_LOCAL_DOMAIN: u32 = 0; -pub const F_IS_LOCAL_ADDRESS: u32 = 1; -pub const F_KEY_GET: u32 = 2; -pub const F_KEY_EXISTS: u32 = 3; -pub const F_KEY_SET: u32 = 4; -pub const F_COUNTER_INCR: u32 = 5; -pub const F_COUNTER_GET: u32 = 6; -pub const F_SQL_QUERY: u32 = 7; -pub const F_DNS_QUERY: u32 = 8; - -pub const VARIABLES_MAP: &[(&str, u32)] = &[ - ("rcpt", V_RECIPIENT), - ("rcpt_domain", V_RECIPIENT_DOMAIN), - ("sender", V_SENDER), - ("sender_domain", V_SENDER_DOMAIN), - ("mx", V_MX), - ("helo_domain", V_HELO_DOMAIN), - ("authenticated_as", V_AUTHENTICATED_AS), - ("listener", V_LISTENER), - ("remote_ip", V_REMOTE_IP), - ("local_ip", V_LOCAL_IP), - ("priority", V_PRIORITY), -]; - -pub const FUNCTIONS_MAP: &[(&str, u32, u32)] = &[ - ("is_local_domain", F_IS_LOCAL_DOMAIN, 2), - ("is_local_address", F_IS_LOCAL_ADDRESS, 2), - ("key_get", F_KEY_GET, 2), - ("key_exists", F_KEY_EXISTS, 2), - ("key_set", F_KEY_SET, 3), - ("counter_incr", F_COUNTER_INCR, 3), - ("counter_get", F_COUNTER_GET, 2), - ("dns_query", F_DNS_QUERY, 2), - ("sql_query", F_SQL_QUERY, 3), -]; - -impl SMTP { - pub async fn eval_if TryFrom>, V: ResolveVariable>( - &self, - if_block: &IfBlock, - resolver: &V, - ) -> Option { - if if_block.is_empty() { - return None; - } - - let result = if_block - .eval( - |var_id| resolver.resolve_variable(var_id), - |fnc_id, params| async move { self.eval_fnc(fnc_id, params, &if_block.key).await }, - ) - .await; - - tracing::trace!(context = "eval_if", - property = if_block.key, - result = ?result, - ); - - match result.try_into() { - Ok(value) => Some(value), - Err(_) => None, - } - } - - pub async fn eval_expr TryFrom>, V: ResolveVariable>( - &self, - expr: &Expression, - resolver: &V, - expr_id: &str, - ) -> Option { - if expr.is_empty() { - return None; - } - - let result = expr - .eval( - |var_id| resolver.resolve_variable(var_id), - |fnc_id, params| async move { self.eval_fnc(fnc_id, params, expr_id).await }, - &mut Vec::new(), - ) - .await; - - tracing::trace!(context = "eval_expr", - property = expr_id, - result = ?result, - ); - - match result.try_into() { - Ok(value) => Some(value), - Err(_) => None, - } - } - - async fn eval_fnc<'x>( - &self, - fnc_id: u32, - params: Vec>, - property: &str, - ) -> Variable<'x> { - let mut params = FncParams::new(params); - - match fnc_id { - F_IS_LOCAL_DOMAIN => { - let directory = params.next_as_string(); - let domain = params.next_as_string(); - - self.get_directory_or_default(directory.as_ref()) - .is_local_domain(domain.as_ref()) - .await - .unwrap_or_else(|err| { - tracing::warn!( - context = "eval_if", - event = "error", - property = property, - error = ?err, - "Failed to check if domain is local." - ); - - false - }) - .into() - } - F_IS_LOCAL_ADDRESS => { - let directory = params.next_as_string(); - let address = params.next_as_string(); - - self.get_directory_or_default(directory.as_ref()) - .rcpt(address.as_ref()) - .await - .unwrap_or_else(|err| { - tracing::warn!( - context = "eval_if", - event = "error", - property = property, - error = ?err, - "Failed to check if address is local." - ); - - false - }) - .into() - } - F_KEY_GET => { - let store = params.next_as_string(); - let key = params.next_as_string(); - - self.get_lookup_store(store.as_ref()) - .key_get::(key.into_owned().into_bytes()) - .await - .map(|value| value.map(|v| v.into_inner()).unwrap_or_default()) - .unwrap_or_else(|err| { - tracing::warn!( - context = "eval_if", - event = "error", - property = property, - error = ?err, - "Failed to get key." - ); - - Variable::default() - }) - } - F_KEY_EXISTS => { - let store = params.next_as_string(); - let key = params.next_as_string(); - - self.get_lookup_store(store.as_ref()) - .key_exists(key.into_owned().into_bytes()) - .await - .unwrap_or_else(|err| { - tracing::warn!( - context = "eval_if", - event = "error", - property = property, - error = ?err, - "Failed to get key." - ); - - false - }) - .into() - } - F_KEY_SET => { - let store = params.next_as_string(); - let key = params.next_as_string(); - let value = params.next_as_string(); - - self.get_lookup_store(store.as_ref()) - .key_set( - key.into_owned().into_bytes(), - value.into_owned().into_bytes(), - None, - ) - .await - .map(|_| true) - .unwrap_or_else(|err| { - tracing::warn!( - context = "eval_if", - event = "error", - property = property, - error = ?err, - "Failed to set key." - ); - - false - }) - .into() - } - F_COUNTER_INCR => { - let store = params.next_as_string(); - let key = params.next_as_string(); - let value = params.next_as_integer(); - - self.get_lookup_store(store.as_ref()) - .counter_incr(key.into_owned().into_bytes(), value, None, true) - .await - .map(Variable::Integer) - .unwrap_or_else(|err| { - tracing::warn!( - context = "eval_if", - event = "error", - property = property, - error = ?err, - "Failed to increment counter." - ); - - Variable::default() - }) - } - F_COUNTER_GET => { - let store = params.next_as_string(); - let key = params.next_as_string(); - - self.get_lookup_store(store.as_ref()) - .counter_get(key.into_owned().into_bytes()) - .await - .map(Variable::Integer) - .unwrap_or_else(|err| { - tracing::warn!( - context = "eval_if", - event = "error", - property = property, - error = ?err, - "Failed to increment counter." - ); - - Variable::default() - }) - } - F_DNS_QUERY => self.dns_query(params).await, - F_SQL_QUERY => self.sql_query(params).await, - _ => Variable::default(), - } - } - - pub fn get_directory(&self, name: &str) -> Option<&Arc> { - self.shared.directories.get(name) - } - - pub fn get_directory_or_default(&self, name: &str) -> &Arc { - self.shared.directories.get(name).unwrap_or_else(|| { - tracing::debug!( - context = "get_directory", - event = "error", - directory = name, - "Directory not found, using default." - ); - - &self.shared.default_directory - }) - } - - pub fn get_lookup_store(&self, name: &str) -> &LookupStore { - self.shared.lookup_stores.get(name).unwrap_or_else(|| { - tracing::debug!( - context = "get_lookup_store", - event = "error", - directory = name, - "Store not found, using default." - ); - - &self.shared.default_lookup_store - }) - } - - pub fn get_arc_sealer(&self, name: &str) -> Option<&ArcSealer> { - self.shared - .sealers - .get(name) - .map(|s| s.as_ref()) - .or_else(|| { - tracing::warn!( - context = "get_arc_sealer", - event = "error", - name = name, - "Arc sealer not found." - ); - - None - }) - } - - pub fn get_dkim_signer(&self, name: &str) -> Option<&DkimSigner> { - self.shared - .signers - .get(name) - .map(|s| s.as_ref()) - .or_else(|| { - tracing::warn!( - context = "get_dkim_signer", - event = "error", - name = name, - "DKIM signer not found." - ); - - None - }) - } - - pub fn get_sieve_script(&self, name: &str) -> Option<&Arc> { - self.shared.scripts.get(name).or_else(|| { - tracing::warn!( - context = "get_sieve_script", - event = "error", - name = name, - "Sieve script not found." - ); - - None - }) - } - - pub fn get_relay_host(&self, name: &str) -> Option<&RelayHost> { - self.shared.relay_hosts.get(name).or_else(|| { - tracing::warn!( - context = "get_relay_host", - event = "error", - name = name, - "Remote host not found." - ); - - None - }) - } - - async fn sql_query<'x>(&self, mut arguments: FncParams<'x>) -> Variable<'x> { - let store = self.get_lookup_store(arguments.next_as_string().as_ref()); - let query = arguments.next_as_string(); - - if query.is_empty() { - tracing::warn!( - context = "eval:sql_query", - event = "invalid", - reason = "Empty query string", - ); - return Variable::default(); - } - - // Obtain arguments - let arguments = match arguments.next() { - Variable::Array(l) => l.into_iter().map(to_store_value).collect(), - v => vec![to_store_value(v)], - }; - - // Run query - if query - .as_bytes() - .get(..6) - .map_or(false, |q| q.eq_ignore_ascii_case(b"SELECT")) - { - if let Ok(mut rows) = store.query::(&query, arguments).await { - match rows.rows.len().cmp(&1) { - Ordering::Equal => { - let mut row = rows.rows.pop().unwrap().values; - match row.len().cmp(&1) { - Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { - row.pop().map(into_variable).unwrap() - } - Ordering::Less => Variable::default(), - _ => Variable::Array( - row.into_iter().map(into_variable).collect::>(), - ), - } - } - Ordering::Less => Variable::default(), - Ordering::Greater => rows - .rows - .into_iter() - .map(|r| { - Variable::Array( - r.values.into_iter().map(into_variable).collect::>(), - ) - }) - .collect::>() - .into(), - } - } else { - false.into() - } - } else { - store.query::(&query, arguments).await.is_ok().into() - } - } - - async fn dns_query<'x>(&self, mut arguments: FncParams<'x>) -> Variable<'x> { - let entry = arguments.next_as_string(); - let record_type = arguments.next_as_string(); - - if record_type.eq_ignore_ascii_case("ip") { - match self - .resolvers - .dns - .ip_lookup(entry.as_ref(), IpLookupStrategy::Ipv4thenIpv6, 10) - .await - { - Ok(result) => result - .iter() - .map(|ip| Variable::from(ip.to_string())) - .collect::>() - .into(), - Err(_) => Variable::default(), - } - } else if record_type.eq_ignore_ascii_case("mx") { - match self.resolvers.dns.mx_lookup(entry.as_ref()).await { - Ok(result) => result - .iter() - .flat_map(|mx| { - mx.exchanges.iter().map(|host| { - Variable::String( - host.strip_suffix('.') - .unwrap_or(host.as_str()) - .to_string() - .into(), - ) - }) - }) - .collect::>() - .into(), - Err(_) => Variable::default(), - } - } else if record_type.eq_ignore_ascii_case("txt") { - match self.resolvers.dns.txt_raw_lookup(entry.as_ref()).await { - Ok(result) => Variable::from(String::from_utf8(result).unwrap_or_default()), - Err(_) => Variable::default(), - } - } else if record_type.eq_ignore_ascii_case("ptr") { - if let Ok(addr) = entry.parse::() { - match self.resolvers.dns.ptr_lookup(addr).await { - Ok(result) => result - .iter() - .map(|host| Variable::from(host.to_string())) - .collect::>() - .into(), - Err(_) => Variable::default(), - } - } else { - Variable::default() - } - } else if record_type.eq_ignore_ascii_case("ipv4") { - match self.resolvers.dns.ipv4_lookup(entry.as_ref()).await { - Ok(result) => result - .iter() - .map(|ip| Variable::from(ip.to_string())) - .collect::>() - .into(), - Err(_) => Variable::default(), - } - } else if record_type.eq_ignore_ascii_case("ipv6") { - match self.resolvers.dns.ipv6_lookup(entry.as_ref()).await { - Ok(result) => result - .iter() - .map(|ip| Variable::from(ip.to_string())) - .collect::>() - .into(), - Err(_) => Variable::default(), - } - } else { - Variable::default() - } - } -} - -struct FncParams<'x> { - params: IntoIter>, -} - -impl<'x> FncParams<'x> { - pub fn new(params: Vec>) -> Self { - Self { - params: params.into_iter(), - } - } - - pub fn next_as_string(&mut self) -> Cow<'x, str> { - self.params.next().unwrap().into_string() - } - - pub fn next_as_integer(&mut self) -> i64 { - self.params.next().unwrap().to_integer().unwrap_or_default() - } - - pub fn next(&mut self) -> Variable<'x> { - self.params.next().unwrap() - } -} - -#[derive(Debug)] -struct VariableWrapper(Variable<'static>); - -impl From for VariableWrapper { - fn from(value: i64) -> Self { - VariableWrapper(Variable::Integer(value)) - } -} - -impl Deserialize for VariableWrapper { - fn deserialize(bytes: &[u8]) -> store::Result { - String::deserialize(bytes).map(|v| VariableWrapper(Variable::String(v.into()))) - } -} - -impl From> for VariableWrapper { - fn from(value: store::Value<'static>) -> Self { - VariableWrapper(value.into()) - } -} - -impl VariableWrapper { - pub fn into_inner(self) -> Variable<'static> { - self.0 - } -} - -fn to_store_value(value: Variable) -> Value { - match value { - Variable::String(v) => Value::Text(v), - Variable::Integer(v) => Value::Integer(v), - Variable::Float(v) => Value::Float(v), - v => Value::Text(v.to_string().into_owned().into()), - } -} - -fn into_variable(value: Value) -> Variable { - match value { - Value::Integer(v) => Variable::Integer(v), - Value::Bool(v) => Variable::Integer(i64::from(v)), - Value::Float(v) => Variable::Float(v), - Value::Text(v) => Variable::String(v), - Value::Blob(v) => Variable::String(v.into_owned().into_string().into()), - Value::Null => Variable::default(), - } -} diff --git a/crates/smtp/src/core/management.rs b/crates/smtp/src/core/management.rs index 4f5e412f..f52eb559 100644 --- a/crates/smtp/src/core/management.rs +++ b/crates/smtp/src/core/management.rs @@ -21,8 +21,12 @@ * for more details. */ -use std::{net::IpAddr, str::FromStr, sync::Arc}; +use std::{net::IpAddr, str::FromStr}; +use common::{ + listener::{limiter::InFlight, SessionData, SessionManager, SessionStream}, + AuthResult, +}; use directory::Type; use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full}; use hyper::{ @@ -54,10 +58,7 @@ use store::{ Deserialize, IterateParams, ValueKey, U64_LEN, }; -use utils::{ - listener::{limiter::InFlight, SessionData, SessionManager, SessionStream}, - url_params::UrlParams, -}; +use utils::url_params::UrlParams; use crate::{ queue::{self, ErrorDetails, HostResponse, QueueId, Status}, @@ -149,7 +150,7 @@ impl SessionManager for SmtpAdminSessionManager { ) -> impl std::future::Future + Send { handle_request( session.stream, - self.inner, + self.inner.into(), session.remote_ip, session.in_flight, ) @@ -159,15 +160,11 @@ impl SessionManager for SmtpAdminSessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } - - fn is_ip_blocked(&self, addr: &IpAddr) -> bool { - false - } } async fn handle_request( stream: impl SessionStream, - core: Arc, + core: SMTP, remote_addr: IpAddr, _in_flight: InFlight, ) { @@ -263,11 +260,14 @@ impl SMTP { }) }) { - let todo = "fix"; - /*match self - .shared - .default_directory - .authenticate(&Credentials::Plain { username, secret }, remote_addr, false) + match self + .core + .authenticate( + &self.core.storage.directory, + &Credentials::Plain { username, secret }, + remote_addr, + false, + ) .await { Ok(AuthResult::Success(principal)) if principal.typ == Type::Superuser => { @@ -294,7 +294,7 @@ impl SMTP { "Temporary authentication failure." ); } - }*/ + } } else { tracing::debug!( context = "management", @@ -371,8 +371,9 @@ impl SMTP { let mut total = 0; let mut total_returned = 0; let _ = self - .shared - .default_data_store + .core + .storage + .data .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { @@ -497,7 +498,7 @@ impl SMTP { message .save_changes(self, prev_event.into(), next_event.into()) .await; - let _ = self.queue.tx.send(queue::Event::Reload).await; + let _ = self.inner.queue_tx.send(queue::Event::Reload).await; } ( @@ -618,8 +619,9 @@ impl SMTP { let mut total = 0; let mut total_returned = 0; let _ = self - .shared - .default_data_store + .core + .storage + .data .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { @@ -760,8 +762,9 @@ impl SMTP { let mut total = 0; let mut last_id = 0; let result = self - .shared - .default_data_store + .core + .storage + .data .iterate( IterateParams::new(from_key, to_key) .set_values(filter.is_some()) @@ -829,8 +832,9 @@ impl SMTP { if let Some(report_id) = parse_incoming_report_id(class, report_id) { match &report_id { ReportClass::Tls { .. } => match self - .shared - .default_data_store + .core + .storage + .data .get_value::>>(ValueKey::from( ValueClass::Report(report_id), )) @@ -847,8 +851,9 @@ impl SMTP { Err(err) => err.into_bad_request(), }, ReportClass::Dmarc { .. } => match self - .shared - .default_data_store + .core + .storage + .data .get_value::>>( ValueKey::from(ValueClass::Report(report_id)), ) @@ -865,8 +870,9 @@ impl SMTP { Err(err) => err.into_bad_request(), }, ReportClass::Arf { .. } => match self - .shared - .default_data_store + .core + .storage + .data .get_value::>>(ValueKey::from( ValueClass::Report(report_id), )) @@ -891,12 +897,7 @@ impl SMTP { if let Some(report_id) = parse_incoming_report_id(class, report_id) { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Report(report_id)); - let result = self - .shared - .default_data_store - .write(batch.build()) - .await - .is_ok(); + let result = self.core.storage.data.write(batch.build()).await.is_ok(); ( StatusCode::OK, serde_json::to_string(&Response { data: result }).unwrap_or_default(), diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index 9722f4ce..aaf91f28 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -28,143 +28,94 @@ use std::{ time::{Duration, Instant}, }; -use ahash::AHashMap; +use common::{ + config::smtp::auth::VerifyStrategy, + listener::{ + limiter::{ConcurrencyLimiter, InFlight}, + ServerInstance, + }, + Core, DeliveryEvent, SharedCore, +}; use dashmap::DashMap; use directory::Directory; -use mail_auth::{common::lru::LruCache, IprevOutput, Resolver, SpfOutput}; -use sieve::{runtime::Variable, Runtime, Sieve}; -use smtp_proto::{ - request::receiver::{ - BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, - RequestReceiver, - }, - IntoString, +use mail_auth::{IprevOutput, SpfOutput}; +use smtp_proto::request::receiver::{ + BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver, }; -use store::{BlobStore, LookupStore, Store, Value}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, }; use tokio_rustls::TlsConnector; use tracing::Span; -use utils::{ - expr, - ipc::DeliveryEvent, - listener::{ - limiter::{ConcurrencyLimiter, InFlight}, - stream::NullIo, - ServerInstance, TcpAcceptor, - }, - snowflake::SnowflakeIdGenerator, -}; +use utils::snowflake::SnowflakeIdGenerator; use crate::{ - config::{ - scripts::SieveContext, ArcSealer, DkimSigner, MailAuthConfig, QueueConfig, RelayHost, - ReportConfig, SessionConfig, VerifyStrategy, - }, inbound::auth::SaslToken, - outbound::{ - dane::{DnssecResolver, Tlsa}, - mta_sts, - }, queue::{self, DomainPart, QueueId}, reporting, }; use self::throttle::{ThrottleKey, ThrottleKeyHasherBuilder}; -pub mod eval; pub mod management; pub mod params; pub mod throttle; pub mod worker; +#[derive(Clone)] +pub struct SmtpInstance { + pub inner: Arc, + pub core: SharedCore, +} + +impl SmtpInstance { + pub fn new(core: SharedCore, inner: impl Into>) -> Self { + Self { + core, + inner: inner.into(), + } + } +} + #[derive(Clone)] pub struct SmtpSessionManager { - pub inner: Arc, + pub inner: SmtpInstance, } #[derive(Clone)] pub struct SmtpAdminSessionManager { - pub inner: Arc, + pub inner: SmtpInstance, } impl SmtpSessionManager { - pub fn new(inner: Arc) -> Self { + pub fn new(inner: SmtpInstance) -> Self { Self { inner } } } impl SmtpAdminSessionManager { - pub fn new(inner: Arc) -> Self { + pub fn new(inner: SmtpInstance) -> Self { Self { inner } } } +#[derive(Clone)] pub struct SMTP { + pub core: Arc, + pub inner: Arc, +} + +pub struct Inner { pub worker_pool: rayon::ThreadPool, - pub session: SessionCore, - pub queue: QueueCore, - pub resolvers: Resolvers, - pub mail_auth: MailAuthConfig, - pub report: ReportCore, - pub sieve: SieveCore, - pub shared: Shared, - #[cfg(feature = "local_delivery")] - pub delivery_tx: mpsc::Sender, -} - -pub struct Shared { - pub scripts: AHashMap>, - pub signers: AHashMap>, - pub sealers: AHashMap>, - pub directories: AHashMap>, - pub lookup_stores: AHashMap, - pub relay_hosts: AHashMap, - - // Default store and directory - pub default_directory: Arc, - pub default_data_store: Store, - pub default_blob_store: BlobStore, - pub default_lookup_store: LookupStore, -} - -pub struct SieveCore { - pub runtime: Runtime, - pub from_addr: String, - pub from_name: String, - pub return_path: String, - pub sign: Vec>, -} - -pub struct Resolvers { - pub dns: Resolver, - pub dnssec: DnssecResolver, - pub cache: DnsCache, -} - -pub struct DnsCache { - pub tlsa: LruCache>, - pub mta_sts: LruCache>, -} - -pub struct SessionCore { - pub config: SessionConfig, - pub throttle: DashMap, -} - -pub struct QueueCore { - pub config: QueueConfig, - pub throttle: DashMap, - pub tx: mpsc::Sender, + pub session_throttle: DashMap, + pub queue_throttle: DashMap, + pub queue_tx: mpsc::Sender, + pub report_tx: mpsc::Sender, pub snowflake_id: SnowflakeIdGenerator, pub connectors: TlsConnectors, -} - -pub struct ReportCore { - pub config: ReportConfig, - pub tx: mpsc::Sender, + #[cfg(feature = "local_delivery")] + pub delivery_tx: mpsc::Sender, } pub struct TlsConnectors { @@ -184,9 +135,10 @@ pub enum State { } pub struct Session { + pub hostname: String, pub state: State, pub instance: Arc, - pub core: Arc, + pub core: SMTP, pub span: Span, pub stream: T, pub data: SessionData, @@ -296,57 +248,12 @@ impl SessionData { } } -pub trait ResolveVariable { - fn resolve_variable(&self, variable: u32) -> expr::Variable<'_>; -} - -pub fn into_sieve_value(value: Value) -> Variable { - match value { - Value::Integer(v) => Variable::Integer(v), - Value::Bool(v) => Variable::Integer(i64::from(v)), - Value::Float(v) => Variable::Float(v), - Value::Text(v) => Variable::String(v.into_owned().into()), - Value::Blob(v) => Variable::String(v.into_owned().into_string().into()), - Value::Null => Variable::default(), - } -} - -pub fn into_store_value(value: Variable) -> Value<'static> { - match value { - Variable::String(v) => Value::Text(v.to_string().into()), - Variable::Integer(v) => Value::Integer(v), - Variable::Float(v) => Value::Float(v), - v => Value::Text(v.to_string().into_owned().into()), - } -} - -pub fn to_store_value(value: &Variable) -> Value<'static> { - match value { - Variable::String(v) => Value::Text(v.to_string().into()), - Variable::Integer(v) => Value::Integer(*v), - Variable::Float(v) => Value::Float(*v), - v => Value::Text(v.to_string().into_owned().into()), - } -} - impl Default for State { fn default() -> Self { State::Request(RequestReceiver::default()) } } -impl VerifyStrategy { - #[inline(always)] - pub fn verify(&self) -> bool { - matches!(self, VerifyStrategy::Strict | VerifyStrategy::Relaxed) - } - - #[inline(always)] - pub fn is_strict(&self) -> bool { - matches!(self, VerifyStrategy::Strict) - } -} - impl PartialEq for SessionAddress { fn eq(&self, other: &Self) -> bool { self.address_lcase == other.address_lcase @@ -376,29 +283,32 @@ impl PartialOrd for SessionAddress { } } +impl From for SMTP { + fn from(value: SmtpInstance) -> Self { + SMTP { + core: value.core.load().clone(), + inner: value.inner, + } + } +} + #[cfg(feature = "local_delivery")] lazy_static::lazy_static! { -static ref SIEVE: Arc = Arc::new(utils::listener::ServerInstance { +static ref SIEVE: Arc = Arc::new(ServerInstance { id: "sieve".to_string(), - listener_id: u16::MAX, - protocol: utils::config::ServerProtocol::Lmtp, - hostname: "localhost".to_string(), - data: "localhost".to_string(), - acceptor: TcpAcceptor::Plain, - limiter: utils::listener::limiter::ConcurrencyLimiter::new(0), + protocol: common::config::server::ServerProtocol::Lmtp, + acceptor: common::listener::TcpAcceptor::Plain, + limiter: ConcurrencyLimiter::new(0), shutdown_rx: tokio::sync::watch::channel(false).1, proxy_networks: vec![], }); } #[cfg(feature = "local_delivery")] -impl Session { - pub fn local( - core: std::sync::Arc, - instance: std::sync::Arc, - data: SessionData, - ) -> Self { +impl Session { + pub fn local(core: SMTP, instance: std::sync::Arc, data: SessionData) -> Self { Session { + hostname: "localhost".to_string(), state: State::None, instance, core, @@ -417,7 +327,7 @@ impl Session { "nrcpt" = data.rcpt_to.len(), "size" = data.message.len(), ), - stream: NullIo::default(), + stream: common::listener::stream::NullIo::default(), data, params: SessionParameters { timeout: Default::default(), @@ -434,9 +344,9 @@ impl Session { rcpt_dsn: Default::default(), max_message_size: Default::default(), auth_match_sender: false, - iprev: crate::config::VerifyStrategy::Disable, - spf_ehlo: crate::config::VerifyStrategy::Disable, - spf_mail_from: crate::config::VerifyStrategy::Disable, + iprev: VerifyStrategy::Disable, + spf_ehlo: VerifyStrategy::Disable, + spf_mail_from: VerifyStrategy::Disable, can_expn: false, can_vrfy: false, }, @@ -445,7 +355,7 @@ impl Session { } pub fn sieve( - core: std::sync::Arc, + core: SMTP, mail_from: SessionAddress, rcpt_to: Vec, message: Vec, diff --git a/crates/smtp/src/core/params.rs b/crates/smtp/src/core/params.rs index c0da42ef..063e7bba 100644 --- a/crates/smtp/src/core/params.rs +++ b/crates/smtp/src/core/params.rs @@ -23,117 +23,171 @@ use std::time::Duration; +use common::config::smtp::auth::VerifyStrategy; use tokio::io::{AsyncRead, AsyncWrite}; -use crate::config::VerifyStrategy; - use super::Session; impl Session { pub async fn eval_session_params(&mut self) { - let c = &self.core.session.config; + let c = &self.core.core.smtp.session; self.data.bytes_left = self + .core .core .eval_if(&c.transfer_limit, self) .await .unwrap_or(250 * 1024 * 1024); self.data.valid_until += self + .core .core .eval_if(&c.duration, self) .await .unwrap_or_else(|| Duration::from_secs(15 * 60)); self.params.timeout = self + .core .core .eval_if(&c.timeout, self) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); self.params.spf_ehlo = self .core - .eval_if(&self.core.mail_auth.spf.verify_ehlo, self) + .core + .eval_if(&self.core.core.smtp.mail_auth.spf.verify_ehlo, self) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.spf_mail_from = self .core - .eval_if(&self.core.mail_auth.spf.verify_mail_from, self) + .core + .eval_if(&self.core.core.smtp.mail_auth.spf.verify_mail_from, self) .await .unwrap_or(VerifyStrategy::Relaxed); self.params.iprev = self .core - .eval_if(&self.core.mail_auth.iprev.verify, self) + .core + .eval_if(&self.core.core.smtp.mail_auth.iprev.verify, self) .await .unwrap_or(VerifyStrategy::Relaxed); // Ehlo parameters - let ec = &self.core.session.config.ehlo; - self.params.ehlo_require = self.core.eval_if(&ec.require, self).await.unwrap_or(true); + let ec = &self.core.core.smtp.session.ehlo; + self.params.ehlo_require = self + .core + .core + .eval_if(&ec.require, self) + .await + .unwrap_or(true); self.params.ehlo_reject_non_fqdn = self + .core .core .eval_if(&ec.reject_non_fqdn, self) .await .unwrap_or(true); // Auth parameters - let ac = &self.core.session.config.auth; + let ac = &self.core.core.smtp.session.auth; self.params.auth_directory = self + .core .core .eval_if::(&ac.directory, self) .await - .and_then(|name| self.core.get_directory(&name)) + .and_then(|name| self.core.core.get_directory(&name)) .cloned(); - self.params.auth_require = self.core.eval_if(&ac.require, self).await.unwrap_or(false); - self.params.auth_errors_max = self.core.eval_if(&ac.errors_max, self).await.unwrap_or(3); + self.params.auth_require = self + .core + .core + .eval_if(&ac.require, self) + .await + .unwrap_or(false); + self.params.auth_errors_max = self + .core + .core + .eval_if(&ac.errors_max, self) + .await + .unwrap_or(3); self.params.auth_errors_wait = self + .core .core .eval_if(&ac.errors_wait, self) .await .unwrap_or_else(|| Duration::from_secs(30)); self.params.auth_plain_text = self + .core .core .eval_if(&ac.allow_plain_text, self) .await .unwrap_or(false); self.params.auth_match_sender = self + .core .core .eval_if(&ac.must_match_sender, self) .await .unwrap_or(true); // VRFY/EXPN parameters - let ec = &self.core.session.config.extensions; - self.params.can_expn = self.core.eval_if(&ec.expn, self).await.unwrap_or(false); - self.params.can_vrfy = self.core.eval_if(&ec.vrfy, self).await.unwrap_or(false); + let ec = &self.core.core.smtp.session.extensions; + self.params.can_expn = self + .core + .core + .eval_if(&ec.expn, self) + .await + .unwrap_or(false); + self.params.can_vrfy = self + .core + .core + .eval_if(&ec.vrfy, self) + .await + .unwrap_or(false); } pub async fn eval_post_auth_params(&mut self) { // Refresh VRFY/EXPN parameters - let ec = &self.core.session.config.extensions; - self.params.can_expn = self.core.eval_if(&ec.expn, self).await.unwrap_or(false); - self.params.can_vrfy = self.core.eval_if(&ec.vrfy, self).await.unwrap_or(false); + let ec = &self.core.core.smtp.session.extensions; + self.params.can_expn = self + .core + .core + .eval_if(&ec.expn, self) + .await + .unwrap_or(false); + self.params.can_vrfy = self + .core + .core + .eval_if(&ec.vrfy, self) + .await + .unwrap_or(false); } pub async fn eval_rcpt_params(&mut self) { - let rc = &self.core.session.config.rcpt; - self.params.rcpt_errors_max = self.core.eval_if(&rc.errors_max, self).await.unwrap_or(10); + let rc = &self.core.core.smtp.session.rcpt; + self.params.rcpt_errors_max = self + .core + .core + .eval_if(&rc.errors_max, self) + .await + .unwrap_or(10); self.params.rcpt_errors_wait = self + .core .core .eval_if(&rc.errors_wait, self) .await .unwrap_or_else(|| Duration::from_secs(30)); self.params.rcpt_max = self + .core .core .eval_if(&rc.max_recipients, self) .await .unwrap_or(100); self.params.rcpt_dsn = self .core - .eval_if(&self.core.session.config.extensions.dsn, self) + .core + .eval_if(&self.core.core.smtp.session.extensions.dsn, self) .await .unwrap_or(true); self.params.max_message_size = self .core - .eval_if(&self.core.session.config.data.max_message_size, self) + .core + .eval_if(&self.core.core.smtp.session.data.max_message_size, self) .await .unwrap_or(25 * 1024 * 1024); } diff --git a/crates/smtp/src/core/throttle.rs b/crates/smtp/src/core/throttle.rs index 0041d65a..79ae1c88 100644 --- a/crates/smtp/src/core/throttle.rs +++ b/crates/smtp/src/core/throttle.rs @@ -21,16 +21,18 @@ * for more details. */ -use ::utils::listener::limiter::ConcurrencyLimiter; +use common::{ + config::smtp::{queue::QueueQuota, *}, + expr::functions::ResolveVariable, + listener::limiter::ConcurrencyLimiter, +}; use dashmap::mapref::entry::Entry; use tokio::io::{AsyncRead, AsyncWrite}; use utils::config::Rate; use std::hash::{BuildHasher, Hash, Hasher}; -use crate::config::*; - -use super::{eval::*, ResolveVariable, Session}; +use super::Session; #[derive(Debug, Clone, Eq)] pub struct ThrottleKey { @@ -81,8 +83,12 @@ impl BuildHasher for ThrottleKeyHasherBuilder { } } -impl QueueQuota { - pub fn new_key(&self, e: &impl ResolveVariable) -> ThrottleKey { +pub trait NewKey: Sized { + fn new_key(&self, e: &impl ResolveVariable) -> ThrottleKey; +} + +impl NewKey for QueueQuota { + fn new_key(&self, e: &impl ResolveVariable) -> ThrottleKey { let mut hasher = blake3::Hasher::new(); if (self.keys & THROTTLE_RCPT) != 0 { @@ -132,8 +138,8 @@ impl QueueQuota { } } -impl Throttle { - pub fn new_key(&self, e: &impl ResolveVariable) -> ThrottleKey { +impl NewKey for Throttle { + fn new_key(&self, e: &impl ResolveVariable) -> ThrottleKey { let mut hasher = blake3::Hasher::new(); if (self.keys & THROTTLE_RCPT) != 0 { @@ -207,16 +213,17 @@ impl Throttle { impl Session { pub async fn is_allowed(&mut self) -> bool { let throttles = if !self.data.rcpt_to.is_empty() { - &self.core.session.config.throttle.rcpt_to + &self.core.core.smtp.session.throttle.rcpt_to } else if self.data.mail_from.is_some() { - &self.core.session.config.throttle.mail_from + &self.core.core.smtp.session.throttle.mail_from } else { - &self.core.session.config.throttle.connect + &self.core.core.smtp.session.throttle.connect }; for t in throttles { if t.expr.is_empty() || self + .core .core .eval_expr(&t.expr, self, "throttle") .await @@ -240,7 +247,7 @@ impl Session { // Check concurrency if let Some(concurrency) = &t.concurrency { - match self.core.session.throttle.entry(key.clone()) { + match self.core.inner.session_throttle.entry(key.clone()) { Entry::Occupied(mut e) => { let limiter = e.get_mut(); if let Some(inflight) = limiter.is_allowed() { @@ -270,8 +277,9 @@ impl Session { if let Some(rate) = &t.rate { if self .core - .shared - .default_lookup_store + .core + .storage + .lookup .is_rate_allowed(key.hash.as_slice(), rate, false) .await .unwrap_or_default() @@ -302,8 +310,9 @@ impl Session { hasher.update(&rate.requests.to_ne_bytes()[..]); self.core - .shared - .default_lookup_store + .core + .storage + .lookup .is_rate_allowed(hasher.finalize().as_bytes(), rate, false) .await .unwrap_or_default() diff --git a/crates/smtp/src/core/worker.rs b/crates/smtp/src/core/worker.rs index 56a030f1..a525b727 100644 --- a/crates/smtp/src/core/worker.rs +++ b/crates/smtp/src/core/worker.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::sync::{atomic::Ordering, Arc}; +use std::sync::atomic::Ordering; use tokio::sync::oneshot; @@ -35,7 +35,7 @@ impl SMTP { { let (tx, rx) = oneshot::channel(); - self.worker_pool.spawn(move || { + self.inner.worker_pool.spawn(move || { tx.send(f()).ok(); }); @@ -53,20 +53,14 @@ impl SMTP { } fn cleanup(&self) { - for throttle in [&self.session.throttle, &self.queue.throttle] { + for throttle in [&self.inner.session_throttle, &self.inner.queue_throttle] { throttle.retain(|_, v| v.concurrent.load(Ordering::Relaxed) > 0); } } -} -pub trait SpawnCleanup { - fn spawn_cleanup(&self); -} - -impl SpawnCleanup for Arc { - fn spawn_cleanup(&self) { + pub fn spawn_cleanup(&self) { let core = self.clone(); - self.worker_pool.spawn(move || { + self.inner.worker_pool.spawn(move || { core.cleanup(); }); } diff --git a/crates/smtp/src/inbound/auth.rs b/crates/smtp/src/inbound/auth.rs index 7682a946..e466faac 100644 --- a/crates/smtp/src/inbound/auth.rs +++ b/crates/smtp/src/inbound/auth.rs @@ -21,6 +21,7 @@ * for more details. */ +use common::AuthResult; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; use smtp_proto::{IntoString, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; @@ -174,15 +175,16 @@ impl Session { } pub async fn authenticate(&mut self, credentials: Credentials) -> Result { - if let Some(lookup) = &self.params.auth_directory { + if let Some(directory) = &self.params.auth_directory { let authenticated_as = match &credentials { Credentials::Plain { username, .. } | Credentials::XOauth2 { username, .. } | Credentials::OAuthBearer { token: username } => username.to_string(), }; - let todo = "fix"; - /*match lookup - .authenticate(&credentials, self.data.remote_ip, false) + match self + .core + .core + .authenticate(directory, &credentials, self.data.remote_ip, false) .await { Ok(AuthResult::Success(principal)) => { @@ -227,7 +229,7 @@ impl Session { return Err(()); } _ => (), - }*/ + } } else { tracing::warn!( parent: &self.span, diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index 7a363bca..8c2ef8f6 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -28,6 +28,9 @@ use std::{ time::{Duration, SystemTime}, }; +use common::{ + config::smtp::auth::VerifyStrategy, listener::SessionStream, scripts::ScriptModification, +}; use mail_auth::{ common::{headers::HeaderWriter, verify::VerifySignature}, dmarc, AuthenticatedMessage, AuthenticationResults, DkimResult, DmarcResult, ReceivedSpf, @@ -39,17 +42,15 @@ use smtp_proto::{ }; use store::write::now; use tokio::{io::AsyncWriteExt, process::Command}; -use utils::{config::Rate, listener::SessionStream}; +use utils::config::Rate; use crate::{ - config::VerifyStrategy, core::{Session, SessionAddress, State}, queue::{self, Message, SimpleEnvelope}, - reporting::analysis::AnalyzeReport, - scripts::{ScriptModification, ScriptResult}, + scripts::ScriptResult, }; -use super::AuthResult; +use super::{ArcSeal, AuthResult, DkimSign}; impl Session { pub async fn queue_message(&mut self) -> Cow<'static, [u8]> { @@ -67,11 +68,12 @@ impl Session { }; // Loop detection - let dc = &self.core.session.config.data; - let ac = &self.core.mail_auth; - let rc = &self.core.report.config; + let dc = &self.core.core.smtp.session.data; + let ac = &self.core.core.smtp.mail_auth; + let rc = &self.core.core.smtp.report; if auth_message.received_headers_count() > self + .core .core .eval_if(&dc.max_received_headers, self) .await @@ -89,24 +91,33 @@ impl Session { // Verify DKIM let dkim = self + .core .core .eval_if(&ac.dkim.verify, self) .await .unwrap_or(VerifyStrategy::Relaxed); let dmarc = self + .core .core .eval_if(&ac.dmarc.verify, self) .await .unwrap_or(VerifyStrategy::Relaxed); let dkim_output = if dkim.verify() || dmarc.verify() { - let dkim_output = self.core.resolvers.dns.verify_dkim(&auth_message).await; + let dkim_output = self + .core + .core + .smtp + .resolvers + .dns + .verify_dkim(&auth_message) + .await; let rejected = dkim.is_strict() && !dkim_output .iter() .any(|d| matches!(d.result(), DkimResult::Pass)); // Send reports for failed signatures - if let Some(rate) = self.core.eval_if::(&rc.dkim.send, self).await { + if let Some(rate) = self.core.core.eval_if::(&rc.dkim.send, self).await { for output in &dkim_output { if let Some(rcpt) = output.failure_report_addr() { self.send_dkim_report(rcpt, &auth_message, &rate, rejected, output) @@ -148,17 +159,26 @@ impl Session { // Verify ARC let arc = self + .core .core .eval_if(&ac.arc.verify, self) .await .unwrap_or(VerifyStrategy::Relaxed); let arc_sealer = self + .core .core .eval_if::(&ac.arc.seal, self) .await - .and_then(|name| self.core.get_arc_sealer(&name)); + .and_then(|name| self.core.core.get_arc_sealer(&name)); let arc_output = if arc.verify() || arc_sealer.is_some() { - let arc_output = self.core.resolvers.dns.verify_arc(&auth_message).await; + let arc_output = self + .core + .core + .smtp + .resolvers + .dns + .verify_arc(&auth_message) + .await; if arc.is_strict() && !matches!(arc_output.result(), DkimResult::Pass | DkimResult::None) @@ -191,7 +211,7 @@ impl Session { // Build authentication results header let mail_from = self.data.mail_from.as_ref().unwrap(); - let mut auth_results = AuthenticationResults::new(&self.instance.hostname); + let mut auth_results = AuthenticationResults::new(&self.hostname); if !dkim_output.is_empty() { auth_results = auth_results.with_dkim_results(&dkim_output, auth_message.from()) } @@ -219,6 +239,8 @@ impl Session { Some(spf_output) if dmarc.verify() => { let dmarc_output = self .core + .core + .smtp .resolvers .dns .verify_dmarc( @@ -339,9 +361,15 @@ impl Session { // Pipe message for pipe in &dc.pipe_commands { - if let Some(command_) = self.core.eval_if::(&pipe.command, self).await { + if let Some(command_) = self + .core + .core + .eval_if::(&pipe.command, self) + .await + { let piped_message = edited_message.as_ref().unwrap_or(&raw_message).clone(); let timeout = self + .core .core .eval_if(&pipe.timeout, self) .await @@ -349,6 +377,7 @@ impl Session { let mut command = Command::new(&command_); for argument in self + .core .core .eval_if::, _>(&pipe.arguments, self) .await @@ -436,10 +465,11 @@ impl Session { // Sieve filtering let mut headers = Vec::with_capacity(64); if let Some(script) = self + .core .core .eval_if::(&dc.script, self) .await - .and_then(|name| self.core.get_sieve_script(&name)) + .and_then(|name| self.core.core.get_sieve_script(&name)) { let params = self .build_script_parameters("data") @@ -536,6 +566,7 @@ impl Session { // Add Received header if self + .core .core .eval_if(&dc.add_received, self) .await @@ -546,6 +577,7 @@ impl Session { // Add authentication results header if self + .core .core .eval_if(&dc.add_auth_results, self) .await @@ -557,6 +589,7 @@ impl Session { // Add Received-SPF header if let Some(spf_output) = &self.data.spf_mail_from { if self + .core .core .eval_if(&dc.add_received_spf, self) .await @@ -567,7 +600,7 @@ impl Session { self.data.remote_ip, &self.data.helo_domain, &message.return_path, - &self.instance.hostname, + &self.hostname, ) .write_header(&mut headers); } @@ -594,7 +627,12 @@ impl Session { // Add any missing headers if !auth_message.has_date_header() - && self.core.eval_if(&dc.add_date, self).await.unwrap_or(true) + && self + .core + .core + .eval_if(&dc.add_date, self) + .await + .unwrap_or(true) { headers.extend_from_slice(b"Date: "); headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); @@ -602,18 +640,20 @@ impl Session { } if !auth_message.has_message_id_header() && self + .core .core .eval_if(&dc.add_message_id, self) .await .unwrap_or(true) { headers.extend_from_slice(b"Message-ID: "); - let _ = generate_message_id_header(&mut headers, &self.instance.hostname); + let _ = generate_message_id_header(&mut headers, &self.hostname); headers.extend_from_slice(b"\r\n"); } // Add Return-Path if self + .core .core .eval_if(&dc.add_return_path, self) .await @@ -627,12 +667,13 @@ impl Session { // DKIM sign let raw_message = edited_message.unwrap_or(raw_message); for signer in self + .core .core .eval_if::, _>(&ac.dkim.sign, self) .await .unwrap_or_default() { - if let Some(signer) = self.core.get_dkim_signer(&signer) { + if let Some(signer) = self.core.core.get_dkim_signer(&signer) { match signer.sign_chained(&[headers.as_ref(), &raw_message]) { Ok(signature) => { signature.write_header(&mut headers); @@ -686,7 +727,7 @@ impl Session { .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); let mut message = Message { - id: self.core.queue.snowflake_id.generate().unwrap_or(created), + id: self.core.inner.snowflake_id.generate().unwrap_or(created), created, return_path: mail_from.address, return_path_lcase: mail_from.address_lcase, @@ -720,8 +761,9 @@ impl Session { }; // Set expiration and notification times - let config = &self.core.queue.config; + let config = &self.core.core.smtp.queue; let (num_intervals, next_notify) = self + .core .core .eval_if::, _>(&config.notify, &envelope) .await @@ -733,6 +775,7 @@ impl Session { now() + future_release.as_secs() + self + .core .core .eval_if(&config.expire, &envelope) .await @@ -746,6 +789,7 @@ impl Session { ) } else { let expire = self + .core .core .eval_if(&config.expire, &envelope) .await @@ -809,7 +853,8 @@ impl Session { if self.data.messages_sent < self .core - .eval_if(&self.core.session.config.data.max_messages, self) + .core + .eval_if(&self.core.core.smtp.session.data.max_messages, self) .await .unwrap_or(10) { @@ -856,7 +901,7 @@ impl Session { headers.extend_from_slice(b")\r\n\t"); } headers.extend_from_slice(b"by "); - headers.extend_from_slice(self.instance.hostname.as_bytes()); + headers.extend_from_slice(self.hostname.as_bytes()); headers.extend_from_slice(b" (Stalwart SMTP) with "); headers.extend_from_slice( match (self.stream.is_tls(), self.data.authenticated_as.is_empty()) { diff --git a/crates/smtp/src/inbound/ehlo.rs b/crates/smtp/src/inbound/ehlo.rs index 71bc1680..f8b11226 100644 --- a/crates/smtp/src/inbound/ehlo.rs +++ b/crates/smtp/src/inbound/ehlo.rs @@ -23,10 +23,10 @@ use std::time::{Duration, SystemTime}; -use crate::{config::session::Mechanism, core::Session, scripts::ScriptResult}; +use crate::{core::Session, scripts::ScriptResult}; +use common::{config::smtp::session::Mechanism, listener::SessionStream}; use mail_auth::spf::verify::HasLabels; use smtp_proto::*; -use utils::listener::SessionStream; impl Session { pub async fn handle_ehlo(&mut self, domain: String, is_extended: bool) -> Result<(), ()> { @@ -50,13 +50,11 @@ impl Session { if self.params.spf_ehlo.verify() { let spf_output = self .core + .core + .smtp .resolvers .dns - .verify_spf_helo( - self.data.remote_ip, - &self.data.helo_domain, - &self.instance.hostname, - ) + .verify_spf_helo(self.data.remote_ip, &self.data.helo_domain, &self.hostname) .await; tracing::debug!(parent: &self.span, @@ -82,9 +80,10 @@ impl Session { // Sieve filtering if let Some(script) = self .core - .eval_if::(&self.core.session.config.ehlo.script, self) + .core + .eval_if::(&self.core.core.smtp.session.ehlo.script, self) .await - .and_then(|name| self.core.get_sieve_script(&name)) + .and_then(|name| self.core.core.get_sieve_script(&name)) { if let ScriptResult::Reject(message) = self .run_script(script.clone(), self.build_script_parameters("ehlo")) @@ -117,22 +116,23 @@ impl Session { if !is_extended { return self - .write(format!("250 {} says hello\r\n", self.instance.hostname).as_bytes()) + .write(format!("250 {} says hello\r\n", self.hostname).as_bytes()) .await; } - let mut response = EhloResponse::new(self.instance.hostname.as_str()); + let mut response = EhloResponse::new(self.hostname.as_str()); response.capabilities = EXT_ENHANCED_STATUS_CODES | EXT_8BIT_MIME | EXT_BINARY_MIME | EXT_SMTP_UTF8; if !self.stream.is_tls() { response.capabilities |= EXT_START_TLS; } - let ec = &self.core.session.config.extensions; - let ac = &self.core.session.config.auth; - let dc = &self.core.session.config.data; + let ec = &self.core.core.smtp.session.extensions; + let ac = &self.core.core.smtp.session.auth; + let dc = &self.core.core.smtp.session.data; // Pipelining if self + .core .core .eval_if(&ec.pipelining, self) .await @@ -142,22 +142,41 @@ impl Session { } // Chunking - if self.core.eval_if(&ec.chunking, self).await.unwrap_or(true) { + if self + .core + .core + .eval_if(&ec.chunking, self) + .await + .unwrap_or(true) + { response.capabilities |= EXT_CHUNKING; } // Address Expansion - if self.core.eval_if(&ec.expn, self).await.unwrap_or(false) { + if self + .core + .core + .eval_if(&ec.expn, self) + .await + .unwrap_or(false) + { response.capabilities |= EXT_EXPN; } // Recipient Verification - if self.core.eval_if(&ec.vrfy, self).await.unwrap_or(false) { + if self + .core + .core + .eval_if(&ec.vrfy, self) + .await + .unwrap_or(false) + { response.capabilities |= EXT_VRFY; } // Require TLS if self + .core .core .eval_if(&ec.requiretls, self) .await @@ -167,13 +186,14 @@ impl Session { } // DSN - if self.core.eval_if(&ec.dsn, self).await.unwrap_or(false) { + if self.core.core.eval_if(&ec.dsn, self).await.unwrap_or(false) { response.capabilities |= EXT_DSN; } // Authentication if self.data.authenticated_as.is_empty() { response.auth_mechanisms = self + .core .core .eval_if::(&ac.mechanisms, self) .await @@ -191,6 +211,7 @@ impl Session { // Future release if let Some(value) = self + .core .core .eval_if::(&ec.future_release, self) .await @@ -205,13 +226,19 @@ impl Session { } // Deliver By - if let Some(value) = self.core.eval_if::(&ec.deliver_by, self).await { + if let Some(value) = self + .core + .core + .eval_if::(&ec.deliver_by, self) + .await + { response.capabilities |= EXT_DELIVER_BY; response.deliver_by = value.as_secs(); } // Priority if let Some(value) = self + .core .core .eval_if::(&ec.mt_priority, self) .await @@ -222,6 +249,7 @@ impl Session { // Size response.size = self + .core .core .eval_if(&dc.max_message_size, self) .await @@ -232,6 +260,7 @@ impl Session { // No soliciting if let Some(value) = self + .core .core .eval_if::(&ec.no_soliciting, self) .await diff --git a/crates/smtp/src/inbound/mail.rs b/crates/smtp/src/inbound/mail.rs index 5a144cd7..7f8ca864 100644 --- a/crates/smtp/src/inbound/mail.rs +++ b/crates/smtp/src/inbound/mail.rs @@ -23,14 +23,15 @@ use std::time::{Duration, SystemTime}; +use common::{listener::SessionStream, scripts::ScriptModification}; use mail_auth::{IprevOutput, IprevResult, SpfOutput, SpfResult}; use smtp_proto::{MailFrom, MtPriority, MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; -use utils::{config::Rate, listener::SessionStream}; +use utils::config::Rate; use crate::{ core::{Session, SessionAddress}, queue::DomainPart, - scripts::{ScriptModification, ScriptResult}, + scripts::ScriptResult, }; impl Session { @@ -54,6 +55,8 @@ impl Session { } else if self.data.iprev.is_none() && self.params.iprev.verify() { let iprev = self .core + .core + .smtp .resolvers .dns .verify_iprev(self.data.remote_ip) @@ -126,9 +129,10 @@ impl Session { // Sieve filtering if let Some(script) = self .core - .eval_if::(&self.core.session.config.mail.script, self) + .core + .eval_if::(&self.core.core.smtp.session.mail.script, self) .await - .and_then(|name| self.core.get_sieve_script(&name)) + .and_then(|name| self.core.core.get_sieve_script(&name)) { match self .run_script(script.clone(), self.build_script_parameters("mail")) @@ -164,7 +168,8 @@ impl Session { // Address rewriting if let Some(new_address) = self .core - .eval_if::(&self.core.session.config.mail.rewrite, self) + .core + .eval_if::(&self.core.core.smtp.session.mail.rewrite, self) .await { let mail_from = self.data.mail_from.as_mut().unwrap(); @@ -180,10 +185,11 @@ impl Session { } // Validate parameters - let config = &self.core.session.config.extensions; - let config_data = &self.core.session.config.data; + let config = &self.core.core.smtp.session.extensions; + let config_data = &self.core.core.smtp.session.data; if (from.flags & MAIL_REQUIRETLS) != 0 && !self + .core .core .eval_if(&config.requiretls, self) .await @@ -196,6 +202,7 @@ impl Session { } if (from.flags & (MAIL_BY_NOTIFY | MAIL_BY_RETURN)) != 0 { if let Some(duration) = self + .core .core .eval_if::(&config.deliver_by, self) .await @@ -225,6 +232,7 @@ impl Session { } if from.mt_priority != 0 { if self + .core .core .eval_if::(&config.mt_priority, self) .await @@ -246,6 +254,7 @@ impl Session { if from.size > 0 && from.size > self + .core .core .eval_if(&config_data.max_message_size, self) .await @@ -258,6 +267,7 @@ impl Session { } if from.hold_for != 0 || from.hold_until != 0 { if let Some(max_hold) = self + .core .core .eval_if::(&config.future_release, self) .await @@ -295,7 +305,14 @@ impl Session { .await; } } - if has_dsn && !self.core.eval_if(&config.dsn, self).await.unwrap_or(false) { + if has_dsn + && !self + .core + .core + .eval_if(&config.dsn, self) + .await + .unwrap_or(false) + { self.data.mail_from = None; return self .write(b"501 5.5.4 DSN extension has been disabled.\r\n") @@ -308,25 +325,29 @@ impl Session { let mail_from = self.data.mail_from.as_ref().unwrap(); let spf_output = if !mail_from.address.is_empty() { self.core + .core + .smtp .resolvers .dns .check_host( self.data.remote_ip, &mail_from.domain, &self.data.helo_domain, - &self.instance.hostname, + &self.hostname, &mail_from.address_lcase, ) .await } else { self.core + .core + .smtp .resolvers .dns .check_host( self.data.remote_ip, &self.data.helo_domain, &self.data.helo_domain, - &self.instance.hostname, + &self.hostname, &format!("postmaster@{}", self.data.helo_domain), ) .await @@ -392,7 +413,8 @@ impl Session { if let (Some(recipient), Some(rate)) = ( spf_output.report_address(), self.core - .eval_if::(&self.core.report.config.spf.send, self) + .core + .eval_if::(&self.core.core.smtp.report.spf.send, self) .await, ) { self.send_spf_report(recipient, &rate, !result, spf_output) diff --git a/crates/smtp/src/inbound/milter/client.rs b/crates/smtp/src/inbound/milter/client.rs index 1daa3c1b..fbbd0de3 100644 --- a/crates/smtp/src/inbound/milter/client.rs +++ b/crates/smtp/src/inbound/milter/client.rs @@ -21,6 +21,7 @@ * for more details. */ +use common::config::smtp::session::Milter; use rustls_pki_types::ServerName; use tokio::{ io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, @@ -28,8 +29,6 @@ use tokio::{ }; use tokio_rustls::{client::TlsStream, TlsConnector}; -use crate::config::Milter; - use super::{ protocol::{SMFIC_CONNECT, SMFIC_HELO, SMFIC_MAIL, SMFIC_RCPT}, receiver::{FrameResult, Receiver}, @@ -115,8 +114,8 @@ impl MilterClient { pub async fn init(&mut self) -> super::Result { self.write(Command::OptionNegotiation(Options { version: match self.version { - Version::V2 => 2, - Version::V6 => 6, + MilterVersion::V2 => 2, + MilterVersion::V6 => 6, }, actions: self.flags_actions, protocol: self.flags_protocol, @@ -264,7 +263,7 @@ impl MilterClient { } pub async fn data(&mut self) -> super::Result { - if matches!(self.version, Version::V6) && !self.has_option(SMFIP_NODATA) { + if matches!(self.version, MilterVersion::V6) && !self.has_option(SMFIP_NODATA) { self.write(Command::Data).await?; if !self.has_option(SMFIP_NR_DATA) { return self.read().await?.into_action(); @@ -367,7 +366,7 @@ impl MilterClient { self.options & opt == opt } - pub fn with_version(mut self, version: Version) -> Self { + pub fn with_version(mut self, version: MilterVersion) -> Self { self.version = version; self } diff --git a/crates/smtp/src/inbound/milter/message.rs b/crates/smtp/src/inbound/milter/message.rs index dda15017..0531e348 100644 --- a/crates/smtp/src/inbound/milter/message.rs +++ b/crates/smtp/src/inbound/milter/message.rs @@ -23,13 +23,12 @@ use std::borrow::Cow; +use common::{config::smtp::session::Milter, listener::SessionStream}; use mail_auth::AuthenticatedMessage; use smtp_proto::request::parser::Rfc5321Parser; use tokio::io::{AsyncRead, AsyncWrite}; -use utils::listener::SessionStream; use crate::{ - config::Milter, core::{Session, SessionAddress, SessionData}, inbound::milter::MilterClient, queue::DomainPart, @@ -48,7 +47,7 @@ impl Session { &self, message: &AuthenticatedMessage<'_>, ) -> Result, Cow<'static, [u8]>> { - let milters = &self.core.session.config.data.milters; + let milters = &self.core.core.smtp.session.data.milters; if milters.is_empty() { return Ok(Vec::new()); } @@ -56,6 +55,7 @@ impl Session { let mut modifications = Vec::new(); for milter in milters { if !self + .core .core .eval_if(&milter.enable, self) .await @@ -143,9 +143,9 @@ impl Session { client .into_tls( if !milter.tls_allow_invalid_certs { - &self.core.queue.connectors.pki_verify + &self.core.inner.connectors.pki_verify } else { - &self.core.queue.connectors.dummy_verify + &self.core.inner.connectors.dummy_verify }, &milter.hostname, ) @@ -178,7 +178,7 @@ impl Session { self.data.remote_port, Macros::new() .with_daemon_name(DAEMON_NAME) - .with_local_hostname(&self.instance.hostname) + .with_local_hostname(&self.hostname) .with_client_address(self.data.remote_ip) .with_client_port(self.data.remote_port) .with_client_ptr(client_ptr.map(|p| p.as_str()).unwrap_or("unknown")), diff --git a/crates/smtp/src/inbound/milter/mod.rs b/crates/smtp/src/inbound/milter/mod.rs index eb798fc9..5a57f36a 100644 --- a/crates/smtp/src/inbound/milter/mod.rs +++ b/crates/smtp/src/inbound/milter/mod.rs @@ -23,6 +23,7 @@ use std::{borrow::Cow, fmt::Display, net::IpAddr, time::Duration}; +use common::config::smtp::session::MilterVersion; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; @@ -41,7 +42,7 @@ pub struct MilterClient { timeout_cmd: Duration, timeout_data: Duration, receiver: Receiver, - version: Version, + version: MilterVersion, options: u32, flags_actions: u32, flags_protocol: u32, @@ -123,12 +124,6 @@ pub enum Action { ConnectionFailure, } -#[derive(Clone, Copy)] -pub enum Version { - V2, - V6, -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub enum Modification { ChangeFrom { diff --git a/crates/smtp/src/inbound/mod.rs b/crates/smtp/src/inbound/mod.rs index 9e2bd9b5..1e0d9e2a 100644 --- a/crates/smtp/src/inbound/mod.rs +++ b/crates/smtp/src/inbound/mod.rs @@ -21,13 +21,12 @@ * for more details. */ +use common::config::smtp::auth::{ArcSealer, DkimSigner}; use mail_auth::{ arc::ArcSet, dkim::Signature, dmarc::Policy, ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimResult, DmarcResult, IprevResult, SpfResult, }; -use crate::config::{ArcSealer, DkimSigner}; - pub mod auth; pub mod data; pub mod ehlo; @@ -38,8 +37,17 @@ pub mod session; pub mod spawn; pub mod vrfy; -impl ArcSealer { - pub fn seal<'x>( +pub trait ArcSeal { + fn seal<'x>( + &self, + message: &'x AuthenticatedMessage, + results: &'x AuthenticationResults, + arc_output: &'x ArcOutput, + ) -> mail_auth::Result>; +} + +impl ArcSeal for ArcSealer { + fn seal<'x>( &self, message: &'x AuthenticatedMessage, results: &'x AuthenticationResults, @@ -52,14 +60,19 @@ impl ArcSealer { } } -impl DkimSigner { - pub fn sign(&self, message: &[u8]) -> mail_auth::Result { +pub trait DkimSign { + fn sign(&self, message: &[u8]) -> mail_auth::Result; + fn sign_chained(&self, message: &[&[u8]]) -> mail_auth::Result; +} + +impl DkimSign for DkimSigner { + fn sign(&self, message: &[u8]) -> mail_auth::Result { match self { DkimSigner::RsaSha256(signer) => signer.sign(message), DkimSigner::Ed25519Sha256(signer) => signer.sign(message), } } - pub fn sign_chained(&self, message: &[&[u8]]) -> mail_auth::Result { + fn sign_chained(&self, message: &[&[u8]]) -> mail_auth::Result { match self { DkimSigner::RsaSha256(signer) => signer.sign_chained(message.iter().copied()), DkimSigner::Ed25519Sha256(signer) => signer.sign_chained(message.iter().copied()), diff --git a/crates/smtp/src/inbound/rcpt.rs b/crates/smtp/src/inbound/rcpt.rs index 08e95122..7365ce5b 100644 --- a/crates/smtp/src/inbound/rcpt.rs +++ b/crates/smtp/src/inbound/rcpt.rs @@ -21,15 +21,15 @@ * for more details. */ +use common::{listener::SessionStream, scripts::ScriptModification}; use smtp_proto::{ RcptTo, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; -use utils::listener::SessionStream; use crate::{ core::{Session, SessionAddress}, queue::DomainPart, - scripts::{ScriptModification, ScriptResult}, + scripts::ScriptResult, }; impl Session { @@ -79,12 +79,13 @@ impl Session { // Address rewriting and Sieve filtering let rcpt_script = self .core - .eval_if::(&self.core.session.config.rcpt.script, self) + .core + .eval_if::(&self.core.core.smtp.session.rcpt.script, self) .await - .and_then(|name| self.core.get_sieve_script(&name)) + .and_then(|name| self.core.core.get_sieve_script(&name)) .cloned(); - if rcpt_script.is_some() || !self.core.session.config.rcpt.rewrite.is_empty() { + if rcpt_script.is_some() || !self.core.core.smtp.session.rcpt.rewrite.is_empty() { // Sieve filtering if let Some(script) = rcpt_script { match self @@ -123,7 +124,8 @@ impl Session { // Address rewriting if let Some(new_address) = self .core - .eval_if::(&self.core.session.config.rcpt.rewrite, self) + .core + .eval_if::(&self.core.core.smtp.session.rcpt.rewrite, self) .await { let rcpt = self.data.rcpt_to.last_mut().unwrap(); @@ -146,13 +148,16 @@ impl Session { let rcpt = self.data.rcpt_to.last().unwrap(); if let Some(directory) = self .core - .eval_if::(&self.core.session.config.rcpt.directory, self) + .core + .eval_if::(&self.core.core.smtp.session.rcpt.directory, self) .await - .and_then(|name| self.core.get_directory(&name)) + .and_then(|name| self.core.core.get_directory(&name)) { if let Ok(is_local_domain) = directory.is_local_domain(&rcpt.domain).await { if is_local_domain { - if let Ok(is_local_address) = directory.rcpt(&rcpt.address_lcase).await { + if let Ok(is_local_address) = + self.core.core.rcpt(directory, &rcpt.address_lcase).await + { if !is_local_address { tracing::debug!(parent: &self.span, context = "rcpt", @@ -179,7 +184,8 @@ impl Session { } } else if !self .core - .eval_if(&self.core.session.config.rcpt.relay, self) + .core + .eval_if(&self.core.core.smtp.session.rcpt.relay, self) .await .unwrap_or(false) { @@ -206,7 +212,8 @@ impl Session { } } else if !self .core - .eval_if(&self.core.session.config.rcpt.relay, self) + .core + .eval_if(&self.core.core.smtp.session.rcpt.relay, self) .await .unwrap_or(false) { diff --git a/crates/smtp/src/inbound/session.rs b/crates/smtp/src/inbound/session.rs index c6e604b2..720e8903 100644 --- a/crates/smtp/src/inbound/session.rs +++ b/crates/smtp/src/inbound/session.rs @@ -21,6 +21,14 @@ * for more details. */ +use common::{ + config::{ + server::ServerProtocol, + smtp::{session::Mechanism, *}, + }, + expr::{self, functions::ResolveVariable}, + listener::SessionStream, +}; use smtp_proto::{ request::receiver::{ BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, @@ -29,12 +37,8 @@ use smtp_proto::{ *, }; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; -use utils::{config::ServerProtocol, listener::SessionStream}; -use crate::{ - config::session::Mechanism, - core::{eval::*, ResolveVariable, Session, State}, -}; +use crate::core::{Session, State}; use super::auth::SaslToken; @@ -94,9 +98,10 @@ impl Session { initial_response, } => { let auth: u64 = self + .core .core .eval_if::( - &self.core.session.config.auth.mechanisms, + &self.core.core.smtp.session.auth.mechanisms, self, ) .await @@ -403,7 +408,7 @@ impl Session { } impl ResolveVariable for Session { - fn resolve_variable(&self, variable: u32) -> utils::expr::Variable<'_> { + fn resolve_variable(&self, variable: u32) -> expr::Variable<'_> { match variable { V_RECIPIENT => self .data @@ -439,7 +444,7 @@ impl ResolveVariable for Session { V_REMOTE_IP => self.data.remote_ip_str.as_str().into(), V_LOCAL_IP => self.data.local_ip_str.as_str().into(), V_PRIORITY => self.data.priority.to_string().into(), - _ => utils::expr::Variable::default(), + _ => expr::Variable::default(), } } } diff --git a/crates/smtp/src/inbound/spawn.rs b/crates/smtp/src/inbound/spawn.rs index 8cd2b178..92172a4b 100644 --- a/crates/smtp/src/inbound/spawn.rs +++ b/crates/smtp/src/inbound/spawn.rs @@ -21,10 +21,10 @@ * for more details. */ -use std::{net::IpAddr, time::Instant}; +use std::time::Instant; +use common::listener::{self, SessionManager, SessionStream}; use tokio_rustls::server::TlsStream; -use utils::listener::{SessionManager, SessionStream}; use crate::{ core::{Session, SessionData, SessionParameters, SmtpSessionManager, State}, @@ -35,11 +35,12 @@ use crate::{ impl SessionManager for SmtpSessionManager { fn handle( self, - session: utils::listener::SessionData, + session: listener::SessionData, ) -> impl std::future::Future + Send { // Create session let mut session = Session { - core: self.inner, + hostname: String::new(), + core: self.inner.into(), instance: session.instance, state: State::default(), span: session.span, @@ -66,20 +67,22 @@ impl SessionManager for SmtpSessionManager { #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async { - let _ = self.inner.queue.tx.send(queue::Event::Stop).await; - let _ = self.inner.report.tx.send(reporting::Event::Stop).await; + let _ = self.inner.inner.queue_tx.send(queue::Event::Stop).await; + let _ = self + .inner + .inner + .report_tx + .send(reporting::Event::Stop) + .await; #[cfg(feature = "local_delivery")] let _ = self + .inner .inner .delivery_tx - .send(utils::ipc::DeliveryEvent::Stop) + .send(common::DeliveryEvent::Stop) .await; } } - - fn is_ip_blocked(&self, addr: &IpAddr) -> bool { - false - } } impl Session { @@ -89,9 +92,10 @@ impl Session { // Sieve filtering if let Some(script) = self .core - .eval_if::(&self.core.session.config.connect.script, self) + .core + .eval_if::(&self.core.core.smtp.session.connect.script, self) .await - .and_then(|name| self.core.get_sieve_script(&name)) + .and_then(|name| self.core.core.get_sieve_script(&name)) { if let ScriptResult::Reject(message) = self .run_script(script.clone(), self.build_script_parameters("connect")) @@ -107,8 +111,31 @@ impl Session { } } - let instance = self.instance.clone(); - if self.write(instance.data.as_bytes()).await.is_err() { + // Obtain hostname + self.hostname = self + .core + .core + .eval_if::(&self.core.core.network.hostname, self) + .await + .unwrap_or_default(); + if self.hostname.is_empty() { + tracing::warn!(parent: &self.span, + context = "connect", + event = "hostname", + "No hostname configured, using 'localhost'." + ); + self.hostname = "locahost".to_string(); + } + + // Obtain greeting + let greeting = self + .core + .core + .eval_if::(&self.core.core.smtp.session.connect.greeting, self) + .await + .unwrap_or_else(|| "Stalwart ESMTP at your service".to_string()); + + if self.write(greeting.as_bytes()).await.is_err() { return false; } @@ -140,7 +167,7 @@ impl Session { } } else if bytes_read > self.data.bytes_left { self - .write(format!("451 4.7.28 {} Session exceeded transfer quota.\r\n", self.instance.hostname).as_bytes()) + .write(format!("451 4.7.28 {} Session exceeded transfer quota.\r\n", self.hostname).as_bytes()) .await .ok(); tracing::debug!( @@ -152,7 +179,7 @@ impl Session { break; } else { self - .write(format!("453 4.3.2 {} Session open for too long.\r\n", self.instance.hostname).as_bytes()) + .write(format!("453 4.3.2 {} Session open for too long.\r\n", self.hostname).as_bytes()) .await .ok(); tracing::debug!( @@ -184,7 +211,7 @@ impl Session { "Connection timed out." ); self - .write(format!("221 2.0.0 {} Disconnecting inactive client.\r\n", self.instance.hostname).as_bytes()) + .write(format!("221 2.0.0 {} Disconnecting inactive client.\r\n", self.hostname).as_bytes()) .await .ok(); break; @@ -210,6 +237,7 @@ impl Session { pub async fn into_tls(self) -> Result>, ()> { let span = self.span; Ok(Session { + hostname: self.hostname, stream: self.instance.tls_accept(self.stream, &span).await?, state: self.state, data: self.data, diff --git a/crates/smtp/src/inbound/vrfy.rs b/crates/smtp/src/inbound/vrfy.rs index 8c8663bf..4cf024b9 100644 --- a/crates/smtp/src/inbound/vrfy.rs +++ b/crates/smtp/src/inbound/vrfy.rs @@ -31,12 +31,18 @@ impl Session { pub async fn handle_vrfy(&mut self, address: String) -> Result<(), ()> { match self .core - .eval_if::(&self.core.session.config.rcpt.directory, self) + .core + .eval_if::(&self.core.core.smtp.session.rcpt.directory, self) .await - .and_then(|name| self.core.get_directory(&name)) + .and_then(|name| self.core.core.get_directory(&name)) { - Some(address_lookup) if self.params.can_vrfy => { - match address_lookup.vrfy(&address.to_lowercase()).await { + Some(directory) if self.params.can_vrfy => { + match self + .core + .core + .vrfy(directory, &address.to_lowercase()) + .await + { Ok(values) if !values.is_empty() => { let mut result = String::with_capacity(32); for (pos, value) in values.iter().enumerate() { @@ -88,12 +94,18 @@ impl Session { pub async fn handle_expn(&mut self, address: String) -> Result<(), ()> { match self .core - .eval_if::(&self.core.session.config.rcpt.directory, self) + .core + .eval_if::(&self.core.core.smtp.session.rcpt.directory, self) .await - .and_then(|name| self.core.get_directory(&name)) + .and_then(|name| self.core.core.get_directory(&name)) { - Some(address_lookup) if self.params.can_expn => { - match address_lookup.expn(&address.to_lowercase()).await { + Some(directory) if self.params.can_expn => { + match self + .core + .core + .expn(directory, &address.to_lowercase()) + .await + { Ok(values) if !values.is_empty() => { let mut result = String::with_capacity(32); for (pos, value) in values.iter().enumerate() { diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs index 43a01746..b2b17bae 100644 --- a/crates/smtp/src/lib.rs +++ b/crates/smtp/src/lib.rs @@ -21,32 +21,17 @@ * for more details. */ -use crate::{ - config::RelayHost, - core::{ - throttle::ThrottleKeyHasherBuilder, QueueCore, ReportCore, SessionCore, TlsConnectors, SMTP, - }, -}; -use std::sync::Arc; +use crate::core::{throttle::ThrottleKeyHasherBuilder, TlsConnectors}; +use core::{Inner, SmtpInstance, SMTP}; -use config::{ - auth::ConfigAuth, queue::ConfigQueue, report::ConfigReport, resolver::ConfigResolver, - scripts::ConfigSieve, session::ConfigSession, shared::ConfigShared, ConfigContext, -}; +use common::SharedCore; use dashmap::DashMap; -use directory::Directories; use mail_send::smtp::tls::build_tls_connector; use queue::manager::SpawnQueue; use reporting::scheduler::SpawnReport; -use store::Stores; use tokio::sync::mpsc; -use utils::{ - config::{Config, ServerProtocol, Servers}, - snowflake::SnowflakeIdGenerator, - UnwrapFailure, -}; +use utils::{config::Config, snowflake::SnowflakeIdGenerator}; -pub mod config; pub mod core; pub mod inbound; pub mod outbound; @@ -59,103 +44,60 @@ pub static DAEMON_NAME: &str = concat!("Stalwart SMTP v", env!("CARGO_PKG_VERSIO impl SMTP { pub async fn init( - config: &Config, - servers: &Servers, - stores: &Stores, - directory: &Directories, - #[cfg(feature = "local_delivery")] delivery_tx: mpsc::Sender, - ) -> Result, String> { - // Read configuration parameters - let mut config_ctx = ConfigContext::new(); - config_ctx.directory = directory.clone(); - config_ctx.stores = stores.clone(); - - // Parse configuration - config.parse_signatures(&mut config_ctx)?; - let sieve_config = config.parse_sieve(&mut config_ctx)?; - let session_config = config.parse_session_config()?; - let queue_config = config.parse_queue()?; - let mail_auth_config = config.parse_mail_auth()?; - let report_config = config.parse_reports()?; - let mut shared = config.parse_shared(&config_ctx)?; - - // Add local delivery host - #[cfg(feature = "local_delivery")] - { - shared.relay_hosts.insert( - "local".to_string(), - RelayHost { - address: String::new(), - port: 0, - protocol: ServerProtocol::Jmap, - tls_implicit: Default::default(), - tls_allow_invalid_certs: Default::default(), - auth: None, - }, - ); - } - - // Build core - let capacity = config.property("cache.capacity")?.unwrap_or(2); + config: &mut Config, + core: SharedCore, + #[cfg(feature = "local_delivery")] delivery_tx: mpsc::Sender, + ) -> SmtpInstance { + // Build inner + let capacity = config.property_("cache.capacity").unwrap_or(2); let shard = config - .property::("cache.shard")? + .property_::("cache.shard") .unwrap_or(32) .next_power_of_two() as usize; let (queue_tx, queue_rx) = mpsc::channel(1024); let (report_tx, report_rx) = mpsc::channel(1024); - let core = Arc::new(SMTP { + let inner = Inner { worker_pool: rayon::ThreadPoolBuilder::new() - .num_threads( + .num_threads(std::cmp::max( config - .property::("global.thread-pool")? + .property_::("global.thread-pool") .filter(|v| *v > 0) .unwrap_or_else(num_cpus::get), - ) + 4, + )) .build() .unwrap(), - resolvers: config.build_resolvers().failed("Failed to build resolvers"), - session: SessionCore { - config: session_config, - throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - capacity, - ThrottleKeyHasherBuilder::default(), - shard, - ), + session_throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + capacity, + ThrottleKeyHasherBuilder::default(), + shard, + ), + queue_throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + capacity, + ThrottleKeyHasherBuilder::default(), + shard, + ), + queue_tx, + report_tx, + snowflake_id: config + .property_::("cluster.node-id") + .map(SnowflakeIdGenerator::with_node_id) + .unwrap_or_default(), + connectors: TlsConnectors { + pki_verify: build_tls_connector(false), + dummy_verify: build_tls_connector(true), }, - queue: QueueCore { - config: queue_config, - throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - capacity, - ThrottleKeyHasherBuilder::default(), - shard, - ), - snowflake_id: config - .property::("storage.cluster.node-id")? - .map(SnowflakeIdGenerator::with_node_id) - .unwrap_or_else(SnowflakeIdGenerator::new), - tx: queue_tx, - connectors: TlsConnectors { - pki_verify: build_tls_connector(false), - dummy_verify: build_tls_connector(true), - }, - }, - report: ReportCore { - tx: report_tx, - config: report_config, - }, - mail_auth: mail_auth_config, - sieve: sieve_config, - shared, #[cfg(feature = "local_delivery")] delivery_tx, - }); + }; + let inner = SmtpInstance::new(core, inner); // Spawn queue manager - queue_rx.spawn(core.clone()); + queue_rx.spawn(inner.clone()); // Spawn report manager - report_rx.spawn(core.clone()); + report_rx.spawn(inner.clone()); - Ok(core) + inner } } diff --git a/crates/smtp/src/outbound/dane/dnssec.rs b/crates/smtp/src/outbound/dane/dnssec.rs index 78066080..5bd5de13 100644 --- a/crates/smtp/src/outbound/dane/dnssec.rs +++ b/crates/smtp/src/outbound/dane/dnssec.rs @@ -21,42 +21,29 @@ * for more details. */ +use common::config::smtp::resolver::{Tlsa, TlsaEntry}; use mail_auth::{ common::{lru::DnsCache, resolver::IntoFqdn}, hickory_resolver::{ - config::{ResolverConfig, ResolverOpts}, - error::{ResolveError, ResolveErrorKind}, + error::ResolveErrorKind, proto::{ error::ProtoErrorKind, rr::rdata::tlsa::{CertUsage, Matching, Selector}, }, - AsyncResolver, Name, + Name, }, }; use std::sync::Arc; -use crate::core::Resolvers; +use crate::core::SMTP; -use super::{DnssecResolver, Tlsa, TlsaEntry}; - -impl DnssecResolver { - pub fn with_capacity( - config: ResolverConfig, - options: ResolverOpts, - ) -> Result { - Ok(Self { - resolver: AsyncResolver::tokio(config, options), - }) - } -} - -impl Resolvers { +impl SMTP { pub async fn tlsa_lookup<'x>( &self, key: impl IntoFqdn<'x>, ) -> mail_auth::Result>> { let key = key.into_fqdn(); - if let Some(value) = self.cache.tlsa.get(key.as_ref()) { + if let Some(value) = self.core.smtp.resolvers.cache.tlsa.get(key.as_ref()) { return Ok(Some(value)); } @@ -67,6 +54,9 @@ impl Resolvers { let mut entries = Vec::new(); let tlsa_lookup = match self + .core + .smtp + .resolvers .dnssec .resolver .tlsa_lookup(Name::from_str_relaxed(key.as_ref())?) @@ -117,7 +107,7 @@ impl Resolvers { } } - Ok(Some(self.cache.tlsa.insert( + Ok(Some(self.core.smtp.resolvers.cache.tlsa.insert( key.into_owned(), Arc::new(Tlsa { entries, @@ -135,8 +125,10 @@ impl Resolvers { value: impl Into>, valid_until: std::time::Instant, ) { - self.cache - .tlsa - .insert(key.into_fqdn().into_owned(), value.into(), valid_until); + self.core.smtp.resolvers.cache.tlsa.insert( + key.into_fqdn().into_owned(), + value.into(), + valid_until, + ); } } diff --git a/crates/smtp/src/outbound/dane/mod.rs b/crates/smtp/src/outbound/dane/mod.rs index 55419213..d9674c0b 100644 --- a/crates/smtp/src/outbound/dane/mod.rs +++ b/crates/smtp/src/outbound/dane/mod.rs @@ -21,26 +21,5 @@ * for more details. */ -use mail_auth::hickory_resolver::TokioAsyncResolver; - pub mod dnssec; pub mod verify; - -pub struct DnssecResolver { - pub resolver: TokioAsyncResolver, -} - -#[derive(Debug, Hash, PartialEq, Eq)] -pub struct TlsaEntry { - pub is_end_entity: bool, - pub is_sha256: bool, - pub is_spki: bool, - pub data: Vec, -} - -#[derive(Debug, Hash, PartialEq, Eq)] -pub struct Tlsa { - pub entries: Vec, - pub has_end_entities: bool, - pub has_intermediates: bool, -} diff --git a/crates/smtp/src/outbound/dane/verify.rs b/crates/smtp/src/outbound/dane/verify.rs index 6fe64316..2e6b3b9b 100644 --- a/crates/smtp/src/outbound/dane/verify.rs +++ b/crates/smtp/src/outbound/dane/verify.rs @@ -21,6 +21,7 @@ * for more details. */ +use common::config::smtp::resolver::Tlsa; use rustls_pki_types::CertificateDer; use sha1::Digest; use sha2::{Sha256, Sha512}; @@ -28,10 +29,17 @@ use x509_parser::prelude::{FromDer, X509Certificate}; use crate::queue::{Error, ErrorDetails, Status}; -use super::Tlsa; +pub trait TlsaVerify { + fn verify( + &self, + span: &tracing::Span, + hostname: &str, + certificates: Option<&[CertificateDer<'_>]>, + ) -> Result<(), Status<(), Error>>; +} -impl Tlsa { - pub fn verify( +impl TlsaVerify for Tlsa { + fn verify( &self, span: &tracing::Span, hostname: &str, diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index 4f38bd5b..a2e1701c 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -21,23 +21,25 @@ * for more details. */ -use std::{ - net::{IpAddr, Ipv4Addr, SocketAddr}, - sync::Arc, - time::Duration, +use crate::outbound::dane::verify::TlsaVerify; +use crate::outbound::mta_sts::verify::VerifyPolicy; +use common::config::{ + server::ServerProtocol, + smtp::{queue::RequireOptional, report::AggregateFrequency}, }; - use mail_auth::{ mta_sts::TlsRpt, report::tlsrpt::{FailureDetails, ResultType}, }; use mail_send::SmtpClient; use smtp_proto::MAIL_REQUIRETLS; +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + time::Duration, +}; use store::write::{now, BatchBuilder, QueueClass, QueueEvent, ValueClass}; -use utils::config::ServerProtocol; use crate::{ - config::{AggregateFrequency, RequireOptional, TlsStrategy}, core::SMTP, queue::{ErrorDetails, Message}, reporting::{tls::TlsRptOptions, PolicyType, TlsEvent}, @@ -47,14 +49,14 @@ use super::{ lookup::ToNextHop, mta_sts, session::{read_greeting, say_helo, try_start_tls, SessionParams, StartTlsResult}, - NextHop, + NextHop, TlsStrategy, }; use crate::queue::{ throttle, DeliveryAttempt, Domain, Error, Event, OnHold, QueueEnvelope, Status, }; impl DeliveryAttempt { - pub async fn try_deliver(mut self, core: Arc) { + pub async fn try_deliver(mut self, core: SMTP) { tokio::spawn(async move { // Lock message self.event = if let Some(event) = core.try_lock_event(self.event).await { @@ -73,7 +75,7 @@ impl DeliveryAttempt { due: self.event.due, queue_id: self.event.queue_id, }))); - let _ = core.shared.default_data_store.write(batch.build()).await; + let _ = core.core.storage.data.write(batch.build()).await; return; }; @@ -103,7 +105,7 @@ impl DeliveryAttempt { message .save_changes(&core, self.event.due.into(), due.into()) .await; - if core.queue.tx.send(Event::Reload).await.is_err() { + if core.inner.queue_tx.send(Event::Reload).await.is_err() { tracing::warn!("Channel closed while trying to notify queue manager."); } return; @@ -111,7 +113,7 @@ impl DeliveryAttempt { } else { // All message recipients expired, do not re-queue. (DSN has been already sent) message.remove(&core, self.event.due).await; - if core.queue.tx.send(Event::Reload).await.is_err() { + if core.inner.queue_tx.send(Event::Reload).await.is_err() { tracing::warn!("Channel closed while trying to notify queue manager."); } @@ -119,7 +121,7 @@ impl DeliveryAttempt { } // Throttle sender - for throttle in &core.queue.config.throttle.sender { + for throttle in &core.core.smtp.queue.throttle.sender { if let Err(err) = core .is_allowed(throttle, &message, &mut self.in_flight, &span) .await @@ -150,14 +152,14 @@ impl DeliveryAttempt { } }; - if core.queue.tx.send(event).await.is_err() { + if core.inner.queue_tx.send(event).await.is_err() { tracing::warn!("Channel closed while trying to notify queue manager."); } return; } } - let queue_config = &core.queue.config; + let queue_config = &core.core.smtp.queue; let mut on_hold = Vec::new(); let no_ip = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); @@ -202,17 +204,18 @@ impl DeliveryAttempt { // Obtain next hop let (mut remote_hosts, is_smtp) = match core + .core .eval_if::(&queue_config.next_hop, &envelope) .await - .and_then(|name| core.get_relay_host(&name)) + .and_then(|name| core.core.get_relay_host(&name)) { #[cfg(feature = "local_delivery")] - Some(next_hop) if next_hop.protocol == ServerProtocol::Jmap => { + Some(next_hop) if next_hop.protocol == ServerProtocol::Http => { // Deliver message locally let delivery_result = message .deliver_local( recipients.iter_mut().filter(|r| r.domain_idx == domain_idx), - &core.delivery_tx, + &core.inner.delivery_tx, &span, ) .await; @@ -221,6 +224,7 @@ impl DeliveryAttempt { domain.set_status( delivery_result, &core + .core .eval_if::, _>(&queue_config.retry, &envelope) .await .unwrap_or_else(|| vec![Duration::from_secs(60)]), @@ -238,19 +242,22 @@ impl DeliveryAttempt { let mut disable_tls = false; let mut tls_strategy = TlsStrategy { mta_sts: core + .core .eval_if(&queue_config.tls.mta_sts, &envelope) .await .unwrap_or(RequireOptional::Optional), ..Default::default() }; let allow_invalid_certs = core + .core .eval_if(&queue_config.tls.invalid_certs, &envelope) .await .unwrap_or(false); // Obtain TLS reporting let tls_report = match core - .eval_if(&core.report.config.tls.send, &envelope) + .core + .eval_if(&core.core.smtp.report.tls.send, &envelope) .await .unwrap_or(AggregateFrequency::Never) { @@ -260,6 +267,8 @@ impl DeliveryAttempt { if is_smtp => { match core + .core + .smtp .resolvers .dns .txt_lookup::(format!("_smtp._tls.{}.", envelope.domain)) @@ -292,7 +301,8 @@ impl DeliveryAttempt { match core .lookup_mta_sts_policy( envelope.domain, - core.eval_if(&queue_config.timeout.mta_sts, &envelope) + core.core + .eval_if(&queue_config.timeout.mta_sts, &envelope) .await .unwrap_or_else(|| Duration::from_secs(10 * 60)), ) @@ -353,6 +363,7 @@ impl DeliveryAttempt { domain.set_status( err, &core + .core .eval_if::, _>(&queue_config.retry, &envelope) .await .unwrap_or_else(|| vec![Duration::from_secs(60)]), @@ -379,7 +390,7 @@ impl DeliveryAttempt { let mx_list; if is_smtp && remote_hosts.is_empty() { // Lookup MX - mx_list = match core.resolvers.dns.mx_lookup(&domain.domain).await { + mx_list = match core.core.smtp.resolvers.dns.mx_lookup(&domain.domain).await { Ok(mx) => mx, Err(err) => { tracing::info!( @@ -391,6 +402,7 @@ impl DeliveryAttempt { domain.set_status( err, &core + .core .eval_if::, _>(&queue_config.retry, &envelope) .await .unwrap_or_else(|| vec![Duration::from_secs(60)]), @@ -401,7 +413,8 @@ impl DeliveryAttempt { if let Some(remote_hosts_) = mx_list.to_remote_hosts( &domain.domain, - core.eval_if(&queue_config.max_mx, &envelope) + core.core + .eval_if(&queue_config.max_mx, &envelope) .await .unwrap_or(5), ) { @@ -418,6 +431,7 @@ impl DeliveryAttempt { "Domain does not accept messages (null MX)".to_string(), )), &core + .core .eval_if::, _>(&queue_config.retry, &envelope) .await .unwrap_or_else(|| vec![Duration::from_secs(60)]), @@ -428,6 +442,7 @@ impl DeliveryAttempt { // Try delivering message let max_multihomed = core + .core .eval_if(&queue_config.max_multihomed, &envelope) .await .unwrap_or(2); @@ -491,21 +506,19 @@ impl DeliveryAttempt { // Update TLS strategy tls_strategy.dane = core + .core .eval_if(&queue_config.tls.dane, &envelope) .await .unwrap_or(RequireOptional::Optional); tls_strategy.tls = core + .core .eval_if(&queue_config.tls.start, &envelope) .await .unwrap_or(RequireOptional::Optional); // Lookup DANE policy let dane_policy = if tls_strategy.try_dane() && is_smtp { - match core - .resolvers - .tlsa_lookup(format!("_25._tcp.{}.", envelope.mx)) - .await - { + match core.tlsa_lookup(format!("_25._tcp.{}.", envelope.mx)).await { Ok(Some(tlsa)) => { if tlsa.has_end_entities { tracing::debug!( @@ -664,6 +677,7 @@ impl DeliveryAttempt { // Connect let conn_timeout = core + .core .eval_if(&queue_config.timeout.connect, &envelope) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); @@ -709,6 +723,7 @@ impl DeliveryAttempt { // Obtain session parameters let local_hostname = core + .core .eval_if::(&queue_config.hostname, &envelope) .await .unwrap_or_else(|| "localhost".to_string()); @@ -720,18 +735,22 @@ impl DeliveryAttempt { hostname: envelope.mx, local_hostname: &local_hostname, timeout_ehlo: core + .core .eval_if(&queue_config.timeout.ehlo, &envelope) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), timeout_mail: core + .core .eval_if(&queue_config.timeout.mail, &envelope) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), timeout_rcpt: core + .core .eval_if(&queue_config.timeout.rcpt, &envelope) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), timeout_data: core + .core .eval_if(&queue_config.timeout.data, &envelope) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)), @@ -744,14 +763,15 @@ impl DeliveryAttempt { || dane_policy.is_some(); let tls_connector = if allow_invalid_certs || remote_host.allow_invalid_certs() { - &core.queue.connectors.dummy_verify + &core.inner.connectors.dummy_verify } else { - &core.queue.connectors.pki_verify + &core.inner.connectors.pki_verify }; let delivery_result = if !remote_host.implicit_tls() { // Read greeting smtp_client.timeout = core + .core .eval_if(&queue_config.timeout.greeting, &envelope) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); @@ -789,6 +809,7 @@ impl DeliveryAttempt { // Try starting TLS if tls_strategy.try_start_tls() && !domain.disable_tls { smtp_client.timeout = core + .core .eval_if(&queue_config.timeout.tls, &envelope) .await .unwrap_or_else(|| Duration::from_secs(3 * 60)); @@ -981,6 +1002,7 @@ impl DeliveryAttempt { } else { // Start TLS smtp_client.timeout = core + .core .eval_if(&queue_config.timeout.tls, &envelope) .await .unwrap_or_else(|| Duration::from_secs(3 * 60)); @@ -1003,6 +1025,7 @@ impl DeliveryAttempt { // Read greeting smtp_client.timeout = core + .core .eval_if(&queue_config.timeout.greeting, &envelope) .await .unwrap_or_else(|| Duration::from_secs(5 * 60)); @@ -1034,6 +1057,7 @@ impl DeliveryAttempt { domain.set_status( delivery_result, &core + .core .eval_if::, _>(&queue_config.retry, &envelope) .await .unwrap_or_else(|| vec![Duration::from_secs(60)]), @@ -1047,6 +1071,7 @@ impl DeliveryAttempt { domain.set_status( last_status, &core + .core .eval_if::, _>(&queue_config.retry, &envelope) .await .unwrap_or_else(|| vec![Duration::from_secs(60)]), @@ -1106,7 +1131,7 @@ impl DeliveryAttempt { Event::Reload }; - if core.queue.tx.send(result).await.is_err() { + if core.inner.queue_tx.send(result).await.is_err() { tracing::warn!( parent: &span, "Channel closed while trying to notify queue manager." diff --git a/crates/smtp/src/outbound/local.rs b/crates/smtp/src/outbound/local.rs index a8962deb..ed105ba1 100644 --- a/crates/smtp/src/outbound/local.rs +++ b/crates/smtp/src/outbound/local.rs @@ -21,9 +21,9 @@ * for more details. */ +use common::{DeliveryEvent, DeliveryResult, IngestMessage}; use smtp_proto::Response; use tokio::sync::{mpsc, oneshot}; -use utils::ipc::{DeliveryEvent, DeliveryResult, IngestMessage}; use crate::queue::{ Error, ErrorDetails, HostResponse, Message, Recipient, Status, RCPT_STATUS_CHANGED, diff --git a/crates/smtp/src/outbound/lookup.rs b/crates/smtp/src/outbound/lookup.rs index 06842581..c8099d52 100644 --- a/crates/smtp/src/outbound/lookup.rs +++ b/crates/smtp/src/outbound/lookup.rs @@ -26,11 +26,12 @@ use std::{ sync::Arc, }; +use common::{config::smtp::V_MX, expr::functions::ResolveVariable}; use mail_auth::{IpLookupStrategy, MX}; use rand::{seq::SliceRandom, Rng}; use crate::{ - core::{eval::V_MX, ResolveVariable, SMTP}, + core::SMTP, queue::{Error, ErrorDetails, Status}, }; @@ -56,7 +57,7 @@ impl SMTP { IpLookupStrategy::Ipv6thenIpv4 => (true, true, false), }; let ipv4_addrs = if has_ipv4 { - match self.resolvers.dns.ipv4_lookup(key).await { + match self.core.smtp.resolvers.dns.ipv4_lookup(key).await { Ok(addrs) => addrs, Err(_) if has_ipv6 => Arc::new(Vec::new()), Err(err) => return Err(err), @@ -66,7 +67,7 @@ impl SMTP { }; if has_ipv6 { - let ipv6_addrs = match self.resolvers.dns.ipv6_lookup(key).await { + let ipv6_addrs = match self.core.smtp.resolvers.dns.ipv6_lookup(key).await { Ok(addrs) => addrs, Err(_) if !ipv4_addrs.is_empty() => Arc::new(Vec::new()), Err(err) => return Err(err), @@ -98,8 +99,8 @@ impl SMTP { } } - pub async fn resolve_host( - &self, + pub async fn resolve_host<'x>( + &'x self, remote_host: &NextHop<'_>, envelope: &impl ResolveVariable, max_multihomed: usize, @@ -107,7 +108,8 @@ impl SMTP { let remote_ips = self .ip_lookup( remote_host.fqdn_hostname().as_ref(), - self.eval_if(&self.queue.config.ip_strategy, envelope) + self.core + .eval_if(&self.core.smtp.queue.ip_strategy, envelope) .await .unwrap_or(IpLookupStrategy::Ipv4thenIpv6), max_multihomed, @@ -136,7 +138,8 @@ impl SMTP { // Obtain source IPv4 address let source_ips = self - .eval_if::, _>(&self.queue.config.source_ip.ipv4, envelope) + .core + .eval_if::, _>(&self.core.smtp.queue.source_ip.ipv4, envelope) .await .unwrap_or_default(); match source_ips.len().cmp(&1) { @@ -153,7 +156,8 @@ impl SMTP { // Obtain source IPv6 address let source_ips = self - .eval_if::, _>(&self.queue.config.source_ip.ipv6, envelope) + .core + .eval_if::, _>(&self.core.smtp.queue.source_ip.ipv6, envelope) .await .unwrap_or_default(); match source_ips.len().cmp(&1) { diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs index d9c33443..9c8e8831 100644 --- a/crates/smtp/src/outbound/mod.rs +++ b/crates/smtp/src/outbound/mod.rs @@ -23,13 +23,15 @@ use std::borrow::Cow; +use common::config::{ + server::ServerProtocol, + smtp::queue::{RelayHost, RequireOptional}, +}; use mail_send::Credentials; use smtp_proto::{Response, Severity}; -use utils::config::ServerProtocol; -use crate::{ - config::RelayHost, - queue::{spool::QueueEventLock, DeliveryAttempt, Error, ErrorDetails, HostResponse, Status}, +use crate::queue::{ + spool::QueueEventLock, DeliveryAttempt, Error, ErrorDetails, HostResponse, Status, }; pub mod dane; @@ -40,6 +42,13 @@ pub mod lookup; pub mod mta_sts; pub mod session; +#[derive(Debug, Clone, Copy, Default)] +pub struct TlsStrategy { + pub dane: RequireOptional, + pub mta_sts: RequireOptional, + pub tls: RequireOptional, +} + impl Status<(), Error> { pub fn from_smtp_error(hostname: &str, command: &str, err: mail_send::Error) -> Self { match err { diff --git a/crates/smtp/src/outbound/mta_sts/lookup.rs b/crates/smtp/src/outbound/mta_sts/lookup.rs index b71e75ab..5e5ddbc6 100644 --- a/crates/smtp/src/outbound/mta_sts/lookup.rs +++ b/crates/smtp/src/outbound/mta_sts/lookup.rs @@ -30,11 +30,12 @@ use std::{ #[cfg(feature = "test_mode")] pub static STS_TEST_POLICY: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); +use common::config::smtp::resolver::Policy; use mail_auth::{common::lru::DnsCache, mta_sts::MtaSts, report::tlsrpt::ResultType}; use crate::core::SMTP; -use super::{Error, Policy}; +use super::{parse::ParsePolicy, Error}; #[allow(unused_variables)] impl SMTP { @@ -45,6 +46,8 @@ impl SMTP { ) -> Result, Error> { // Lookup MTA-STS TXT record let record = match self + .core + .smtp .resolvers .dns .txt_lookup::(format!("_mta-sts.{domain}.")) @@ -53,7 +56,7 @@ impl SMTP { Ok(record) => record, Err(err) => { // Return the cached policy in case of failure - return if let Some(value) = self.resolvers.cache.mta_sts.get(domain) { + return if let Some(value) = self.core.smtp.resolvers.cache.mta_sts.get(domain) { Ok(value) } else { Err(err.into()) @@ -62,7 +65,7 @@ impl SMTP { }; // Check if the policy has been cached - if let Some(value) = self.resolvers.cache.mta_sts.get(domain) { + if let Some(value) = self.core.smtp.resolvers.cache.mta_sts.get(domain) { if value.id == record.id { return Ok(value); } @@ -95,11 +98,11 @@ impl SMTP { 86400 }); - Ok(self - .resolvers - .cache - .mta_sts - .insert(domain.to_string(), Arc::new(policy), valid_until)) + Ok(self.core.smtp.resolvers.cache.mta_sts.insert( + domain.to_string(), + Arc::new(policy), + valid_until, + )) } #[cfg(feature = "test_mode")] @@ -109,7 +112,7 @@ impl SMTP { value: Policy, valid_until: std::time::Instant, ) { - self.resolvers.cache.mta_sts.insert( + self.core.smtp.resolvers.cache.mta_sts.insert( key.into_fqdn().into_owned(), Arc::new(value), valid_until, diff --git a/crates/smtp/src/outbound/mta_sts/mod.rs b/crates/smtp/src/outbound/mta_sts/mod.rs index c032d8cc..a852e822 100644 --- a/crates/smtp/src/outbound/mta_sts/mod.rs +++ b/crates/smtp/src/outbound/mta_sts/mod.rs @@ -25,27 +25,6 @@ pub mod lookup; pub mod parse; pub mod verify; -#[derive(Debug, PartialEq, Eq, Hash)] -pub enum Mode { - Enforce, - Testing, - None, -} - -#[derive(Debug, PartialEq, Eq, Hash)] -pub enum MxPattern { - Equals(String), - StartsWith(String), -} - -#[derive(Debug, PartialEq, Eq, Hash)] -pub struct Policy { - pub id: String, - pub mode: Mode, - pub mx: Vec, - pub max_age: u64, -} - #[derive(Debug)] pub enum Error { Dns(mail_auth::Error), diff --git a/crates/smtp/src/outbound/mta_sts/parse.rs b/crates/smtp/src/outbound/mta_sts/parse.rs index 38eb60d3..abd37e58 100644 --- a/crates/smtp/src/outbound/mta_sts/parse.rs +++ b/crates/smtp/src/outbound/mta_sts/parse.rs @@ -21,10 +21,16 @@ * for more details. */ -use super::{Mode, MxPattern, Policy}; +use common::config::smtp::resolver::{Mode, MxPattern, Policy}; -impl Policy { - pub fn parse(mut data: &str, id: String) -> Result { +pub trait ParsePolicy { + fn parse(data: &str, id: String) -> Result + where + Self: Sized; +} + +impl ParsePolicy for Policy { + fn parse(mut data: &str, id: String) -> Result { let mut mode = Mode::None; let mut max_age: u64 = 86400; let mut mx = Vec::new(); diff --git a/crates/smtp/src/outbound/mta_sts/verify.rs b/crates/smtp/src/outbound/mta_sts/verify.rs index 3b0e95d4..ec6eceae 100644 --- a/crates/smtp/src/outbound/mta_sts/verify.rs +++ b/crates/smtp/src/outbound/mta_sts/verify.rs @@ -21,10 +21,15 @@ * for more details. */ -use super::{Mode, MxPattern, Policy}; +use common::config::smtp::resolver::{Mode, MxPattern, Policy}; -impl Policy { - pub fn verify(&self, mx_host: &str) -> bool { +pub trait VerifyPolicy { + fn verify(&self, mx_host: &str) -> bool; + fn enforce(&self) -> bool; +} + +impl VerifyPolicy for Policy { + fn verify(&self, mx_host: &str) -> bool { if self.mode != Mode::None { for mx_pattern in &self.mx { match mx_pattern { @@ -49,7 +54,7 @@ impl Policy { } } - pub fn enforce(&self) -> bool { + fn enforce(&self) -> bool { self.mode == Mode::Enforce } } diff --git a/crates/smtp/src/outbound/session.rs b/crates/smtp/src/outbound/session.rs index ec679a81..eef41778 100644 --- a/crates/smtp/src/outbound/session.rs +++ b/crates/smtp/src/outbound/session.rs @@ -21,6 +21,7 @@ * for more details. */ +use common::config::smtp::queue::RequireOptional; use mail_send::{smtp::AssertReply, Credentials, SmtpClient}; use smtp_proto::{ EhloResponse, Response, Severity, EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, @@ -36,13 +37,14 @@ use tokio::{ use tokio_rustls::{client::TlsStream, TlsConnector}; use crate::{ - config::{RequireOptional, TlsStrategy}, core::SMTP, queue::{ErrorDetails, HostResponse, RCPT_STATUS_CHANGED}, }; use crate::queue::{Error, Message, Recipient, Status}; +use super::TlsStrategy; + pub struct SessionParams<'x> { pub span: &'x tracing::Span, pub core: &'x SMTP, @@ -535,8 +537,9 @@ pub async fn send_message( ) -> Result<(), Status<(), Error>> { match params .core - .shared - .default_blob_store + .core + .storage + .blob .get_blob(message.blob_hash.as_slice(), 0..usize::MAX) .await { diff --git a/crates/smtp/src/queue/dsn.rs b/crates/smtp/src/queue/dsn.rs index 79765313..16c6a93a 100644 --- a/crates/smtp/src/queue/dsn.rs +++ b/crates/smtp/src/queue/dsn.rs @@ -44,7 +44,7 @@ impl SMTP { pub async fn send_dsn(&self, message: &mut Message, span: &tracing::Span) { if !message.return_path.is_empty() { if let Some(dsn) = message.build_dsn(self, span).await { - let mut dsn_message = self.queue.new_message("", "", ""); + let mut dsn_message = self.new_message("", "", ""); dsn_message .add_recipient_parts( &message.return_path, @@ -56,7 +56,7 @@ impl SMTP { // Sign message let signature = self - .sign_message(message, &self.queue.config.dsn.sign, &dsn, span) + .sign_message(message, &self.core.smtp.queue.dsn.sign, &dsn, span) .await; dsn_message .queue(signature.as_deref(), &dsn, self, span) @@ -70,7 +70,7 @@ impl SMTP { impl Message { pub async fn build_dsn(&mut self, core: &SMTP, span: &tracing::Span) -> Option> { - let config = &core.queue.config; + let config = &core.core.smtp.queue; let now = now(); let mut txt_success = String::new(); @@ -232,6 +232,7 @@ impl Message { let envelope = SimpleEnvelope::new(self, &domain.domain); if let Some(next_notify) = core + .core .eval_if::, _>(&config.notify, &envelope) .await .and_then(|notify| { @@ -250,14 +251,17 @@ impl Message { // Obtain hostname and sender addresses let from_name = core + .core .eval_if(&config.dsn.name, self) .await .unwrap_or_else(|| String::from("Mail Delivery Subsystem")); let from_addr = core + .core .eval_if(&config.dsn.address, self) .await .unwrap_or_else(|| String::from("MAILER-DAEMON@localhost")); let reporting_mta = core + .core .eval_if(&config.hostname, self) .await .unwrap_or_else(|| String::from("localhost")); @@ -269,8 +273,9 @@ impl Message { // Fetch up to 1024 bytes of message headers let headers = match core - .shared - .default_blob_store + .core + .storage + .blob .get_blob(self.blob_hash.as_slice(), 0..1024) .await { diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index 7379f850..6760da3a 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -21,15 +21,12 @@ * for more details. */ -use std::{ - sync::{atomic::Ordering, Arc}, - time::Duration, -}; +use std::{sync::atomic::Ordering, time::Duration}; use store::write::now; use tokio::sync::mpsc; -use crate::core::SMTP; +use crate::core::{SmtpInstance, SMTP}; use super::{spool::QueueEventLock, DeliveryAttempt, Event, Message, OnHold, Status}; @@ -37,13 +34,13 @@ pub(crate) const SHORT_WAIT: Duration = Duration::from_millis(1); pub(crate) const LONG_WAIT: Duration = Duration::from_secs(86400 * 365); pub struct Queue { - pub core: Arc, + pub core: SmtpInstance, pub on_hold: Vec>, pub next_wake_up: Duration, } impl SpawnQueue for mpsc::Receiver { - fn spawn(mut self, core: Arc) { + fn spawn(mut self, core: SmtpInstance) { tokio::spawn(async move { let mut queue = Queue::new(core); @@ -68,7 +65,7 @@ impl SpawnQueue for mpsc::Receiver { } impl Queue { - pub fn new(core: Arc) -> Self { + pub fn new(core: SmtpInstance) -> Self { Queue { core, on_hold: Vec::with_capacity(128), @@ -78,19 +75,20 @@ impl Queue { pub async fn process_events(&mut self) { // Deliver any concurrency limited messages + let core = SMTP::from(self.core.clone()); while let Some(queue_event) = self.next_on_hold() { DeliveryAttempt::new(queue_event) - .try_deliver(self.core.clone()) + .try_deliver(core.clone()) .await; } // Deliver scheduled messages let now = now(); self.next_wake_up = LONG_WAIT; - for queue_event in self.core.next_event().await { + for queue_event in core.next_event().await { if queue_event.due <= now { DeliveryAttempt::new(queue_event) - .try_deliver(self.core.clone()) + .try_deliver(core.clone()) .await; } else { self.next_wake_up = Duration::from_secs(queue_event.due - now); @@ -202,5 +200,5 @@ impl Message { } pub trait SpawnQueue { - fn spawn(self, core: Arc); + fn spawn(self, core: SmtpInstance); } diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 0d4966db..4fa0d864 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -27,15 +27,15 @@ use std::{ time::{Duration, Instant, SystemTime}, }; +use common::{ + config::smtp::*, + expr::{self, functions::ResolveVariable}, + listener::limiter::{ConcurrencyLimiter, InFlight}, +}; use serde::{Deserialize, Serialize}; use smtp_proto::Response; use store::write::now; -use utils::{ - listener::limiter::{ConcurrencyLimiter, InFlight}, - BlobHash, -}; - -use crate::core::{eval::*, ResolveVariable}; +use utils::BlobHash; use self::spool::QueueEventLock; @@ -219,7 +219,7 @@ impl<'x> SimpleEnvelope<'x> { } impl<'x> ResolveVariable for SimpleEnvelope<'x> { - fn resolve_variable(&self, variable: u32) -> utils::expr::Variable<'_> { + fn resolve_variable(&self, variable: u32) -> expr::Variable<'x> { match variable { V_SENDER => self.message.return_path_lcase.as_str().into(), V_SENDER_DOMAIN => self.message.return_path_domain.as_str().into(), @@ -240,7 +240,7 @@ pub struct QueueEnvelope<'x> { } impl<'x> ResolveVariable for QueueEnvelope<'x> { - fn resolve_variable(&self, variable: u32) -> utils::expr::Variable<'x> { + fn resolve_variable(&self, variable: u32) -> expr::Variable<'x> { match variable { V_SENDER => self.message.return_path_lcase.as_str().into(), V_SENDER_DOMAIN => self.message.return_path_domain.as_str().into(), @@ -255,7 +255,7 @@ impl<'x> ResolveVariable for QueueEnvelope<'x> { } impl ResolveVariable for Message { - fn resolve_variable(&self, variable: u32) -> utils::expr::Variable<'_> { + fn resolve_variable(&self, variable: u32) -> expr::Variable<'_> { match variable { V_SENDER => self.return_path_lcase.as_str().into(), V_SENDER_DOMAIN => self.return_path_domain.as_str().into(), @@ -274,7 +274,7 @@ impl<'x> RecipientDomain<'x> { } impl<'x> ResolveVariable for RecipientDomain<'x> { - fn resolve_variable(&self, variable: u32) -> utils::expr::Variable<'_> { + fn resolve_variable(&self, variable: u32) -> expr::Variable<'x> { match variable { V_RECIPIENT_DOMAIN => self.0.into(), _ => "".into(), diff --git a/crates/smtp/src/queue/quota.rs b/crates/smtp/src/queue/quota.rs index c8502128..0a5b58fc 100644 --- a/crates/smtp/src/queue/quota.rs +++ b/crates/smtp/src/queue/quota.rs @@ -21,15 +21,13 @@ * for more details. */ +use common::{config::smtp::queue::QueueQuota, expr::functions::ResolveVariable}; use store::{ write::{BatchBuilder, QueueClass, ValueClass}, ValueKey, }; -use crate::{ - config::QueueQuota, - core::{ResolveVariable, SMTP}, -}; +use crate::core::{throttle::NewKey, SMTP}; use super::{Message, QuotaKey, SimpleEnvelope, Status}; @@ -37,8 +35,8 @@ impl SMTP { pub async fn has_quota(&self, message: &mut Message) -> bool { let mut quota_keys = Vec::new(); - if !self.queue.config.quota.sender.is_empty() { - for quota in &self.queue.config.quota.sender { + if !self.core.smtp.queue.quota.sender.is_empty() { + for quota in &self.core.smtp.queue.quota.sender { if !self .check_quota(quota, message, message.size, 0, &mut quota_keys) .await @@ -48,7 +46,7 @@ impl SMTP { } } - for quota in &self.queue.config.quota.rcpt_domain { + for quota in &self.core.smtp.queue.quota.rcpt_domain { for (pos, domain) in message.domains.iter().enumerate() { if !self .check_quota( @@ -65,7 +63,7 @@ impl SMTP { } } - for quota in &self.queue.config.quota.rcpt { + for quota in &self.core.smtp.queue.quota.rcpt { for (pos, rcpt) in message.recipients.iter().enumerate() { if !self .check_quota( @@ -91,9 +89,9 @@ impl SMTP { true } - async fn check_quota( - &self, - quota: &QueueQuota, + async fn check_quota<'x>( + &'x self, + quota: &'x QueueQuota, envelope: &impl ResolveVariable, size: usize, id: u64, @@ -101,6 +99,7 @@ impl SMTP { ) -> bool { if !quota.expr.is_empty() && self + .core .eval_expr("a.expr, envelope, "check_quota") .await .unwrap_or(false) @@ -108,8 +107,9 @@ impl SMTP { let key = quota.new_key(envelope); if let Some(max_size) = quota.size { let used_size = self - .shared - .default_data_store + .core + .storage + .data .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaSize( key.as_ref().to_vec(), )))) @@ -127,8 +127,9 @@ impl SMTP { if let Some(max_messages) = quota.messages { let total_messages = self - .shared - .default_data_store + .core + .storage + .data .get_counter(ValueKey::from(ValueClass::Queue(QueueClass::QuotaCount( key.as_ref().to_vec(), )))) diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 8b362faf..426d5236 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -29,7 +29,7 @@ use store::write::{now, BatchBuilder, Bincode, BlobOp, QueueClass, QueueEvent, V use store::{Deserialize, IterateParams, Serialize, ValueKey, U64_LEN}; use utils::BlobHash; -use crate::core::{QueueCore, SMTP}; +use crate::core::SMTP; use super::{ Domain, Event, Message, QueueId, QuotaKey, Recipient, Schedule, SimpleEnvelope, Status, @@ -46,7 +46,7 @@ pub struct QueueEventLock { pub lock_expiry: u64, } -impl QueueCore { +impl SMTP { pub fn new_message( &self, return_path: impl Into, @@ -57,7 +57,7 @@ impl QueueCore { .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); Message { - id: self.snowflake_id.generate().unwrap_or(created), + id: self.inner.snowflake_id.generate().unwrap_or(created), created, return_path: return_path.into(), return_path_lcase: return_path_lcase.into(), @@ -72,9 +72,7 @@ impl QueueCore { quota_keys: Vec::new(), } } -} -impl SMTP { pub async fn next_event(&self) -> Vec { let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(QueueEvent { due: 0, @@ -88,8 +86,9 @@ impl SMTP { let mut events = Vec::new(); let now = now(); let result = self - .shared - .default_data_store + .core + .storage + .data .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { @@ -145,7 +144,7 @@ impl SMTP { })), event.lock_expiry.serialize(), ); - match self.shared.default_data_store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => Some(event), Err(store::Error::AssertValueFailed) => { tracing::debug!( @@ -171,8 +170,9 @@ impl SMTP { pub async fn read_message(&self, id: QueueId) -> Option { match self - .shared - .default_data_store + .core + .storage + .data .get_value::>(ValueKey::from(ValueClass::Queue(QueueClass::Message( id, )))) @@ -226,7 +226,7 @@ impl Message { }, 0u32.serialize(), ); - if let Err(err) = core.shared.default_data_store.write(batch.build()).await { + if let Err(err) = core.core.storage.data.write(batch.build()).await { tracing::error!( parent: span, context = "queue", @@ -237,8 +237,9 @@ impl Message { return false; } if let Err(err) = core - .shared - .default_blob_store + .core + .storage + .blob .put_blob(self.blob_hash.as_slice(), message.as_ref()) .await { @@ -303,7 +304,7 @@ impl Message { Bincode::new(self).serialize(), ); - if let Err(err) = core.shared.default_data_store.write(batch.build()).await { + if let Err(err) = core.core.storage.data.write(batch.build()).await { tracing::error!( parent: span, context = "queue", @@ -315,7 +316,7 @@ impl Message { } // Queue the message - if core.queue.tx.send(Event::Reload).await.is_err() { + if core.inner.queue_tx.send(Event::Reload).await.is_err() { tracing::warn!( parent: span, context = "queue", @@ -341,8 +342,9 @@ impl Message { } else { let idx = self.domains.len(); let expires = core + .core .eval_if( - &core.queue.config.expire, + &core.core.smtp.queue.expire, &SimpleEnvelope::new(self, &rcpt_domain), ) .await @@ -418,7 +420,7 @@ impl Message { Bincode::new(self).serialize(), ); - if let Err(err) = core.shared.default_data_store.write(batch.build()).await { + if let Err(err) = core.core.storage.data.write(batch.build()).await { tracing::error!( context = "queue", event = "error", @@ -468,7 +470,7 @@ impl Message { }))) .clear(ValueClass::Queue(QueueClass::Message(self.id))); - if let Err(err) = core.shared.default_data_store.write(batch.build()).await { + if let Err(err) = core.core.storage.data.write(batch.build()).await { tracing::error!( context = "queue", event = "error", diff --git a/crates/smtp/src/queue/throttle.rs b/crates/smtp/src/queue/throttle.rs index 17130cbf..791dfdcc 100644 --- a/crates/smtp/src/queue/throttle.rs +++ b/crates/smtp/src/queue/throttle.rs @@ -21,14 +21,15 @@ * for more details. */ +use common::{ + config::smtp::Throttle, + expr::functions::ResolveVariable, + listener::limiter::{ConcurrencyLimiter, InFlight}, +}; use dashmap::mapref::entry::Entry; use store::write::now; -use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; -use crate::{ - config::Throttle, - core::{ResolveVariable, SMTP}, -}; +use crate::core::{throttle::NewKey, SMTP}; use super::{Domain, Status}; @@ -39,15 +40,16 @@ pub enum Error { } impl SMTP { - pub async fn is_allowed( - &self, - throttle: &Throttle, + pub async fn is_allowed<'x>( + &'x self, + throttle: &'x Throttle, envelope: &impl ResolveVariable, in_flight: &mut Vec, span: &tracing::Span, ) -> Result<(), Error> { if throttle.expr.is_empty() || self + .core .eval_expr(&throttle.expr, envelope, "throttle") .await .unwrap_or(false) @@ -56,8 +58,9 @@ impl SMTP { if let Some(rate) = &throttle.rate { if let Ok(Some(next_refill)) = self - .shared - .default_lookup_store + .core + .storage + .lookup .is_rate_allowed(key.as_ref(), rate, false) .await { @@ -76,7 +79,7 @@ impl SMTP { } if let Some(concurrency) = &throttle.concurrency { - match self.queue.throttle.entry(key) { + match self.inner.queue_throttle.entry(key) { Entry::Occupied(mut e) => { let limiter = e.get_mut(); if let Some(inflight) = limiter.is_allowed() { diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index 252c187b..e11b09cd 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -71,15 +71,11 @@ pub struct IncomingReport { pub report: T, } -pub trait AnalyzeReport { - fn analyze_report(&self, message: Arc>); -} - -impl AnalyzeReport for Arc { - fn analyze_report(&self, message: Arc>) { +impl SMTP { + pub fn analyze_report(&self, message: Arc>) { let core = self.clone(); let handle = Handle::current(); - self.worker_pool.spawn(move || { + self.inner.worker_pool.spawn(move || { let message = if let Some(message) = MessageParser::default().parse(message.as_ref()) { message } else { @@ -284,15 +280,9 @@ impl AnalyzeReport for Arc { }; // Store report - if let Some(expires_in) = &core.report.config.analysis.store { + if let Some(expires_in) = &core.core.smtp.report.analysis.store { let expires = now() + expires_in.as_secs(); - let id = core - .report - .config - .analysis - .report_id - .generate() - .unwrap_or(expires); + let id = core.inner.snowflake_id.generate().unwrap_or(expires); let mut batch = BatchBuilder::new(); match report { @@ -336,7 +326,7 @@ impl AnalyzeReport for Arc { let batch = batch.build(); let _enter = handle.enter(); handle.spawn(async move { - if let Err(err) = core.shared.default_data_store.write(batch).await { + if let Err(err) = core.core.storage.data.write(batch).await { tracing::warn!( context = "report", event = "error", diff --git a/crates/smtp/src/reporting/dkim.rs b/crates/smtp/src/reporting/dkim.rs index 3f6fb0a0..cb4ffa00 100644 --- a/crates/smtp/src/reporting/dkim.rs +++ b/crates/smtp/src/reporting/dkim.rs @@ -57,8 +57,9 @@ impl Session { return; } - let config = &self.core.report.config.dkim; + let config = &self.core.core.smtp.report.dkim; let from_addr = self + .core .core .eval_if(&config.address, self) .await @@ -66,7 +67,7 @@ impl Session { let mut report = Vec::with_capacity(128); self.new_auth_failure(output.result().into(), rejected) .with_authentication_results( - AuthenticationResults::new(&self.instance.hostname) + AuthenticationResults::new(&self.hostname) .with_dkim_result(output, message.from()) .to_string(), ) @@ -77,6 +78,7 @@ impl Session { .write_rfc5322( ( self.core + .core .eval_if(&config.name, self) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) @@ -85,6 +87,7 @@ impl Session { ), rcpt, &self + .core .core .eval_if(&config.subject, self) .await diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index a10f2f8d..360f1020 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -24,6 +24,7 @@ use std::collections::hash_map::Entry; use ahash::AHashMap; +use common::config::smtp::report::AggregateFrequency; use mail_auth::{ common::verify::VerifySignature, dmarc::{self, URI}, @@ -39,12 +40,11 @@ use tokio::io::{AsyncRead, AsyncWrite}; use utils::config::Rate; use crate::{ - config::AggregateFrequency, core::{Session, SMTP}, queue::{DomainPart, RecipientDomain}, }; -use super::{scheduler::ToHash, DmarcEvent, ReportLock, SerializedSize}; +use super::{scheduler::ToHash, AggregateTimestamp, DmarcEvent, ReportLock, SerializedSize}; #[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct DmarcFormat { @@ -65,16 +65,18 @@ impl Session { arc_output: &Option>, ) { let dmarc_record = dmarc_output.dmarc_record_cloned().unwrap(); - let config = &self.core.report.config.dmarc; + let config = &self.core.core.smtp.report.dmarc; // Send failure report if let (Some(failure_rate), Some(report_options)) = ( - self.core.eval_if::(&config.send, self).await, + self.core.core.eval_if::(&config.send, self).await, dmarc_output.failure_report(), ) { // Verify that any external reporting addresses are authorized let rcpts = match self .core + .core + .smtp .resolvers .dns .verify_dmarc_report_address(dmarc_output.domain(), dmarc_record.ruf()) @@ -122,6 +124,7 @@ impl Session { if !rcpts.is_empty() { let mut report = Vec::with_capacity(128); let from_addr = self + .core .core .eval_if(&config.address, self) .await @@ -206,6 +209,7 @@ impl Session { .write_rfc5322( ( self.core + .core .eval_if(&config.name, self) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) @@ -214,6 +218,7 @@ impl Session { ), &rcpts.join(", "), &self + .core .core .eval_if(&config.subject, self) .await @@ -256,7 +261,8 @@ impl Session { // Send agregate reports let interval = self .core - .eval_if(&self.core.report.config.dmarc_aggregate.send, self) + .core + .eval_if(&self.core.core.smtp.report.dmarc_aggregate.send, self) .await .unwrap_or(AggregateFrequency::Never); @@ -310,12 +316,13 @@ impl SMTP { // Generate report let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( - self.eval_if( - &self.report.config.dmarc_aggregate.max_size, - &RecipientDomain::new(event.domain.as_str()), - ) - .await - .unwrap_or(25 * 1024 * 1024), + self.core + .eval_if( + &self.core.smtp.report.dmarc_aggregate.max_size, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or(25 * 1024 * 1024), )); let mut rua = Vec::new(); let report = match self @@ -344,6 +351,8 @@ impl SMTP { // Verify external reporting addresses let rua = match self + .core + .smtp .resolvers .dns .verify_dmarc_report_address(&event.domain, &rua) @@ -381,8 +390,9 @@ impl SMTP { }; // Serialize report - let config = &self.report.config.dmarc_aggregate; + let config = &self.core.smtp.report.dmarc_aggregate; let from_addr = self + .core .eval_if( &config.address, &RecipientDomain::new(event.domain.as_str()), @@ -392,14 +402,16 @@ impl SMTP { let mut message = Vec::with_capacity(2048); let _ = report.write_rfc5322( &self + .core .eval_if( - &self.report.config.submitter, + &self.core.smtp.report.submitter, &RecipientDomain::new(event.domain.as_str()), ) .await .unwrap_or_else(|| "localhost".to_string()), ( - self.eval_if(&config.name, &RecipientDomain::new(event.domain.as_str())) + self.core + .eval_if(&config.name, &RecipientDomain::new(event.domain.as_str())) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) .as_str(), @@ -424,8 +436,9 @@ impl SMTP { ) -> store::Result> { // Deserialize report let dmarc = match self - .shared - .default_data_store + .core + .storage + .data .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::DmarcReportHeader(event.clone()), ))) @@ -439,21 +452,23 @@ impl SMTP { let _ = std::mem::replace(rua, dmarc.rua); // Create report - let config = &self.report.config.dmarc_aggregate; + let config = &self.core.smtp.report.dmarc_aggregate; let mut report = Report::new() .with_policy_published(dmarc.policy) .with_date_range_begin(event.seq_id) .with_date_range_end(event.due) .with_report_id(format!("{}_{}", event.policy_hash, event.seq_id)) .with_email( - self.eval_if( - &config.address, - &RecipientDomain::new(event.domain.as_str()), - ) - .await - .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), + self.core + .eval_if( + &config.address, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), ); if let Some(org_name) = self + .core .eval_if::( &config.org_name, &RecipientDomain::new(event.domain.as_str()), @@ -463,6 +478,7 @@ impl SMTP { report = report.with_org_name(org_name); } if let Some(contact_info) = self + .core .eval_if::( &config.contact_info, &RecipientDomain::new(event.domain.as_str()), @@ -494,8 +510,9 @@ impl SMTP { }, ))); let mut record_map = AHashMap::with_capacity(dmarc.records.len()); - self.shared - .default_data_store + self.core + .storage + .data .iterate( IterateParams::new(from_key, to_key).ascending(), |_, v| match record_map.entry(Bincode::::deserialize(v)?.inner) { @@ -542,8 +559,9 @@ impl SMTP { }; if let Err(err) = self - .shared - .default_data_store + .core + .storage + .data .delete_range( ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent(from_key))), ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent(to_key))), @@ -561,7 +579,7 @@ impl SMTP { let mut batch = BatchBuilder::new(); batch.clear(ValueClass::Queue(QueueClass::DmarcReportHeader(event))); - if let Err(err) = self.shared.default_data_store.write(batch.build()).await { + if let Err(err) = self.core.storage.data.write(batch.build()).await { tracing::warn!( context = "report", event = "error", @@ -584,8 +602,9 @@ impl SMTP { // Write policy if missing let mut builder = BatchBuilder::new(); if self - .shared - .default_data_store + .core + .storage + .data .get_value::<()>(ValueKey::from(ValueClass::Queue( QueueClass::DmarcReportHeader(report_event.clone()), ))) @@ -617,13 +636,13 @@ impl SMTP { } // Write entry - report_event.seq_id = self.queue.snowflake_id.generate().unwrap_or_else(now); + report_event.seq_id = self.inner.snowflake_id.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), Bincode::new(event.report_record).serialize(), ); - if let Err(err) = self.shared.default_data_store.write(builder.build()).await { + if let Err(err) = self.core.storage.data.write(builder.build()).await { tracing::error!( context = "report", event = "error", diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index 70f38416..f0ead1ae 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -23,6 +23,13 @@ use std::{io, sync::Arc, time::SystemTime}; +use common::{ + config::smtp::{ + report::{AddressMatch, AggregateFrequency}, + resolver::{Policy, Tlsa}, + }, + expr::if_block::IfBlock, +}; use mail_auth::{ common::headers::HeaderWriter, dmarc::Dmarc, @@ -35,12 +42,10 @@ use mail_parser::DateTime; use store::write::{QueueClass, ReportEvent}; use tokio::io::{AsyncRead, AsyncWrite}; -use utils::config::if_block::IfBlock; use crate::{ - config::{AddressMatch, AggregateFrequency}, core::{Session, SMTP}, - outbound::{dane::Tlsa, mta_sts::Policy}, + inbound::DkimSign, queue::{DomainPart, Message}, USER_AGENT, }; @@ -93,7 +98,7 @@ impl Session { .map_or(0, |d| d.as_secs()) as i64, ) .with_source_ip(self.data.remote_ip) - .with_reporting_mta(&self.instance.hostname) + .with_reporting_mta(&self.hostname) .with_user_agent(USER_AGENT) .with_delivery_result(if rejected { DeliveryResult::Reject @@ -103,7 +108,7 @@ impl Session { } pub fn is_report(&self) -> bool { - for addr_match in &self.core.report.config.analysis.addresses { + for addr_match in &self.core.core.smtp.report.analysis.addresses { for addr in &self.data.rcpt_to { match addr_match { AddressMatch::StartsWith(prefix) if addr.address_lcase.starts_with(prefix) => { @@ -135,9 +140,7 @@ impl SMTP { // Build message let from_addr_lcase = from_addr.to_lowercase(); let from_addr_domain = from_addr_lcase.domain_part().to_string(); - let mut message = self - .queue - .new_message(from_addr, from_addr_lcase, from_addr_domain); + let mut message = self.new_message(from_addr, from_addr_lcase, from_addr_domain); for rcpt_ in rcpts { message.add_recipient(rcpt_.as_ref(), self).await; } @@ -169,7 +172,7 @@ impl SMTP { } pub async fn schedule_report(&self, report: impl Into) { - if self.report.tx.send(report.into()).await.is_err() { + if self.inner.report_tx.send(report.into()).await.is_err() { tracing::warn!(contex = "report", "Channel send failed."); } } @@ -182,13 +185,14 @@ impl SMTP { span: &tracing::Span, ) -> Option> { let signers = self + .core .eval_if::, _>(config, message) .await .unwrap_or_default(); if !signers.is_empty() { let mut headers = Vec::with_capacity(64); for signer in signers.iter() { - if let Some(signer) = self.get_dkim_signer(signer) { + if let Some(signer) = self.core.get_dkim_signer(signer) { match signer.sign(bytes) { Ok(signature) => { signature.write_header(&mut headers); @@ -210,8 +214,14 @@ impl SMTP { } } -impl AggregateFrequency { - pub fn to_timestamp(&self) -> u64 { +pub trait AggregateTimestamp { + fn to_timestamp(&self) -> u64; + fn to_timestamp_(&self, dt: DateTime) -> u64; + fn as_secs(&self) -> u64; +} + +impl AggregateTimestamp for AggregateFrequency { + fn to_timestamp(&self) -> u64 { self.to_timestamp_(DateTime::from_timestamp( SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) @@ -219,7 +229,7 @@ impl AggregateFrequency { )) } - pub fn to_timestamp_(&self, mut dt: DateTime) -> u64 { + fn to_timestamp_(&self, mut dt: DateTime) -> u64 { (match self { AggregateFrequency::Hourly => { dt.minute = 0; @@ -243,7 +253,7 @@ impl AggregateFrequency { }) as u64 } - pub fn as_secs(&self) -> u64 { + fn as_secs(&self) -> u64 { match self { AggregateFrequency::Hourly => 3600, AggregateFrequency::Daily => 86400, diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index b5bd0098..b2c73a6d 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -22,12 +22,10 @@ */ use ahash::{AHashMap, RandomState}; +use common::Core; use mail_auth::dmarc::Dmarc; -use std::{ - sync::Arc, - time::{Duration, Instant, SystemTime}, -}; +use std::time::{Duration, Instant, SystemTime}; use store::{ write::{now, BatchBuilder, QueueClass, ReportEvent, ValueClass}, Deserialize, IterateParams, Serialize, ValueKey, @@ -35,14 +33,14 @@ use store::{ use tokio::sync::mpsc; use crate::{ - core::{worker::SpawnCleanup, SMTP}, + core::{SmtpInstance, SMTP}, queue::{manager::LONG_WAIT, spool::LOCK_EXPIRY}, }; use super::{Event, ReportLock}; impl SpawnReport for mpsc::Receiver { - fn spawn(mut self, core: Arc) { + fn spawn(mut self, core: SmtpInstance) { tokio::spawn(async move { let mut last_cleanup = Instant::now(); let mut next_wake_up; @@ -50,7 +48,7 @@ impl SpawnReport for mpsc::Receiver { loop { // Read events let now = now(); - let events = core.next_report_event().await; + let events = next_report_event(&core.core.load()).await; next_wake_up = events .last() .and_then(|e| match e { @@ -63,6 +61,7 @@ impl SpawnReport for mpsc::Receiver { }) .unwrap_or(LONG_WAIT); + let core = SMTP::from(core.clone()); let core_ = core.clone(); tokio::spawn(async move { let mut tls_reports = AHashMap::new(); @@ -117,66 +116,67 @@ impl SpawnReport for mpsc::Receiver { } } -impl SMTP { - pub async fn next_report_event(&self) -> Vec { - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( - ReportEvent { - due: 0, - policy_hash: 0, - seq_id: 0, - domain: String::new(), +async fn next_report_event(core: &Core) -> Vec { + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( + ReportEvent { + due: 0, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( + ReportEvent { + due: u64::MAX, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))); + + let mut events = Vec::new(); + let now = now(); + let result = core + .storage + .data + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let event = ReportEvent::deserialize(key)?; + if event.seq_id == 0 { + // Skip lock + return Ok(true); + } + let do_continue = event.due <= now; + events.push(if *key.last().unwrap() == 0 { + QueueClass::DmarcReportHeader(event) + } else { + QueueClass::TlsReportHeader(event) + }); + Ok(do_continue) }, - ))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( - ReportEvent { - due: u64::MAX, - policy_hash: 0, - seq_id: 0, - domain: String::new(), - }, - ))); + ) + .await; - let mut events = Vec::new(); - let now = now(); - let result = self - .shared - .default_data_store - .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - let event = ReportEvent::deserialize(key)?; - if event.seq_id == 0 { - // Skip lock - return Ok(true); - } - let do_continue = event.due <= now; - events.push(if *key.last().unwrap() == 0 { - QueueClass::DmarcReportHeader(event) - } else { - QueueClass::TlsReportHeader(event) - }); - Ok(do_continue) - }, - ) - .await; - - if let Err(err) = result { - tracing::error!( - context = "queue", - event = "error", - "Failed to read from store: {}", - err - ); - } - - events + if let Err(err) = result { + tracing::error!( + context = "queue", + event = "error", + "Failed to read from store: {}", + err + ); } + events +} + +impl SMTP { pub async fn try_lock_report(&self, lock: QueueClass) -> bool { let now = now(); match self - .shared - .default_data_store + .core + .storage + .data .get_value::(ValueKey::from(ValueClass::Queue(lock.clone()))) .await { @@ -188,7 +188,7 @@ impl SMTP { ValueClass::Queue(lock.clone()), (now + LOCK_EXPIRY).serialize(), ); - match self.shared.default_data_store.write(batch.build()).await { + match self.core.storage.data.write(batch.build()).await { Ok(_) => true, Err(store::Error::AssertValueFailed) => { tracing::debug!( @@ -273,5 +273,5 @@ impl ToTimestamp for Duration { } pub trait SpawnReport { - fn spawn(self, core: Arc); + fn spawn(self, core: SmtpInstance); } diff --git a/crates/smtp/src/reporting/spf.rs b/crates/smtp/src/reporting/spf.rs index ac2398d1..196bb43e 100644 --- a/crates/smtp/src/reporting/spf.rs +++ b/crates/smtp/src/reporting/spf.rs @@ -48,8 +48,9 @@ impl Session { } // Generate report - let config = &self.core.report.config.spf; + let config = &self.core.core.smtp.report.spf; let from_addr = self + .core .core .eval_if(&config.address, self) .await @@ -58,14 +59,14 @@ impl Session { self.new_auth_failure(AuthFailureType::Spf, rejected) .with_authentication_results( if let Some(mail_from) = &self.data.mail_from { - AuthenticationResults::new(&self.instance.hostname).with_spf_mailfrom_result( + AuthenticationResults::new(&self.hostname).with_spf_mailfrom_result( output, self.data.remote_ip, &mail_from.address, &self.data.helo_domain, ) } else { - AuthenticationResults::new(&self.instance.hostname).with_spf_ehlo_result( + AuthenticationResults::new(&self.hostname).with_spf_ehlo_result( output, self.data.remote_ip, &self.data.helo_domain, @@ -77,6 +78,7 @@ impl Session { .write_rfc5322( ( self.core + .core .eval_if(&config.name, self) .await .unwrap_or_else(|| "Mailer Daemon".to_string()) @@ -85,6 +87,7 @@ impl Session { ), rcpt, &self + .core .core .eval_if(&config.subject, self) .await diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 827f1087..33a09377 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -24,6 +24,10 @@ use std::{collections::hash_map::Entry, sync::Arc, time::Duration}; use ahash::AHashMap; +use common::config::smtp::{ + report::AggregateFrequency, + resolver::{Mode, MxPattern}, +}; use mail_auth::{ flate2::{write::GzEncoder, Compression}, mta_sts::{ReportUri, TlsRpt}, @@ -40,15 +44,9 @@ use store::{ Deserialize, IterateParams, Serialize, ValueKey, }; -use crate::{ - config::AggregateFrequency, - core::SMTP, - outbound::mta_sts::{Mode, MxPattern}, - queue::RecipientDomain, - USER_AGENT, -}; +use crate::{core::SMTP, queue::RecipientDomain, USER_AGENT}; -use super::{scheduler::ToHash, ReportLock, SerializedSize, TlsEvent}; +use super::{scheduler::ToHash, AggregateTimestamp, ReportLock, SerializedSize, TlsEvent}; #[derive(Debug, Clone)] pub struct TlsRptOptions { @@ -83,12 +81,13 @@ impl SMTP { // Generate report let mut rua = Vec::new(); let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( - self.eval_if( - &self.report.config.tls.max_size, - &RecipientDomain::new(domain_name), - ) - .await - .unwrap_or(25 * 1024 * 1024), + self.core + .eval_if( + &self.core.smtp.report.tls.max_size, + &RecipientDomain::new(domain_name), + ) + .await + .unwrap_or(25 * 1024 * 1024), )); let report = match self .generate_tls_aggregate_report(&events, &mut rua, Some(&mut serialized_size)) @@ -198,8 +197,9 @@ impl SMTP { // Deliver report over SMTP if !rcpts.is_empty() { - let config = &self.report.config.tls; + let config = &self.core.smtp.report.tls; let from_addr = self + .core .eval_if(&config.address, &RecipientDomain::new(domain_name)) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); @@ -207,14 +207,16 @@ impl SMTP { let _ = report.write_rfc5322_from_bytes( domain_name, &self + .core .eval_if( - &self.report.config.submitter, + &self.core.smtp.report.submitter, &RecipientDomain::new(domain_name), ) .await .unwrap_or_else(|| "localhost".to_string()), ( - self.eval_if(&config.name, &RecipientDomain::new(domain_name)) + self.core + .eval_if(&config.name, &RecipientDomain::new(domain_name)) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) .as_str(), @@ -255,9 +257,10 @@ impl SMTP { .first() .map(|e| (e.domain.as_str(), e.seq_id, e.due, e.policy_hash)) .unwrap(); - let config = &self.report.config.tls; + let config = &self.core.smtp.report.tls; let mut report = TlsReport { organization_name: self + .core .eval_if(&config.org_name, &RecipientDomain::new(domain_name)) .await .clone(), @@ -266,6 +269,7 @@ impl SMTP { end_datetime: DateTime::from_timestamp(event_to as i64), }, contact_info: self + .core .eval_if(&config.contact_info, &RecipientDomain::new(domain_name)) .await .clone(), @@ -279,8 +283,9 @@ impl SMTP { for event in events { let tls = if let Some(tls) = self - .shared - .default_data_store + .core + .storage + .data .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::TlsReportHeader(event.clone()), ))) @@ -315,8 +320,9 @@ impl SMTP { domain: event.domain.clone(), }))); let mut record_map = AHashMap::new(); - self.shared - .default_data_store + self.core + .storage + .data .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { if let Some(failure_details) = Bincode::>::deserialize(v)?.inner @@ -394,8 +400,9 @@ impl SMTP { // Write policy if missing let mut builder = BatchBuilder::new(); if self - .shared - .default_data_store + .core + .storage + .data .get_value::<()>(ValueKey::from(ValueClass::Queue( QueueClass::TlsReportHeader(report_event.clone()), ))) @@ -481,13 +488,13 @@ impl SMTP { } // Write entry - report_event.seq_id = self.queue.snowflake_id.generate().unwrap_or_else(now); + report_event.seq_id = self.inner.snowflake_id.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), Bincode::new(event.failure).serialize(), ); - if let Err(err) = self.shared.default_data_store.write(builder.build()).await { + if let Err(err) = self.core.storage.data.write(builder.build()).await { tracing::error!( context = "report", event = "error", @@ -516,8 +523,9 @@ impl SMTP { // Remove report events if let Err(err) = self - .shared - .default_data_store + .core + .storage + .data .delete_range( ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(from_key))), ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(to_key))), @@ -542,7 +550,7 @@ impl SMTP { batch.clear(ValueClass::Queue(QueueClass::TlsReportHeader(event))); } - if let Err(err) = self.shared.default_data_store.write(batch.build()).await { + if let Err(err) = self.core.storage.data.write(batch.build()).await { tracing::warn!( context = "report", event = "error", diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index a79403ac..7d0e4563 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use common::scripts::plugins::PluginContext; use mail_auth::common::headers::HeaderWriter; use sieve::{ compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, @@ -34,9 +35,9 @@ use smtp_proto::{ }; use tokio::runtime::Handle; -use crate::{core::SMTP, queue::DomainPart}; +use crate::{core::SMTP, inbound::DkimSign, queue::DomainPart}; -use super::{plugins::PluginContext, ScriptModification, ScriptParameters, ScriptResult}; +use super::{ScriptModification, ScriptParameters, ScriptResult}; impl SMTP { pub fn run_script_blocking( @@ -48,13 +49,14 @@ impl SMTP { ) -> ScriptResult { // Create filter instance let mut instance = self + .core .sieve - .runtime + .trusted_runtime .filter(params.message.as_deref().map_or(b"", |m| &m[..])) .with_vars_env(params.variables) .with_envelope_list(params.envelope) - .with_user_address(&self.sieve.from_addr) - .with_user_full_name(&self.sieve.from_name); + .with_user_address(&self.core.sieve.from_addr) + .with_user_full_name(&self.core.sieve.from_name); let mut input = Input::script("__script", script); let mut messages: Vec> = Vec::new(); @@ -67,7 +69,7 @@ impl SMTP { match result { Ok(event) => match event { Event::IncludeScript { name, optional } => { - if let Some(script) = self.shared.scripts.get(name.as_str()) { + if let Some(script) = self.core.sieve.scripts.get(name.as_str()) { input = Input::script(name, script.clone()); } else if optional { input = false.into(); @@ -88,7 +90,7 @@ impl SMTP { } => { input = false.into(); 'outer: for list in lists { - if let Some(store) = self.shared.lookup_stores.get(&list) { + if let Some(store) = self.core.storage.lookups.get(&list) { for value in &values { if let Ok(true) = handle.block_on( store.key_exists( @@ -115,12 +117,12 @@ impl SMTP { } } Event::Function { id, arguments } => { - input = self.run_plugin_blocking( + input = self.core.run_plugin_blocking( id, PluginContext { span: &span, handle: &handle, - core: self, + core: &self.core, message: instance.message(), modifications: &mut modifications, arguments, @@ -147,10 +149,10 @@ impl SMTP { message_id, } => { // Build message - let return_path_lcase = self.sieve.return_path.to_lowercase(); + let return_path_lcase = self.core.sieve.return_path.to_lowercase(); let return_path_domain = return_path_lcase.domain_part().to_string(); - let mut message = self.queue.new_message( - self.sieve.return_path.clone(), + let mut message = self.new_message( + self.core.sieve.return_path.clone(), return_path_lcase, return_path_domain, ); @@ -263,18 +265,20 @@ impl SMTP { instance.message().raw_message().into() }; if let Some(raw_message) = raw_message { - let headers = if !self.sieve.sign.is_empty() { + let headers = if !self.core.sieve.sign.is_empty() { let mut headers = Vec::new(); - for dkim in &self.sieve.sign { - match dkim.sign(raw_message) { - Ok(signature) => { - signature.write_header(&mut headers); - } - Err(err) => { - tracing::warn!(parent: &span, - context = "dkim", - event = "sign-failed", - reason = %err); + for dkim in &self.core.sieve.sign { + if let Some(dkim) = self.core.get_dkim_signer(dkim) { + match dkim.sign(raw_message) { + Ok(signature) => { + signature.write_header(&mut headers); + } + Err(err) => { + tracing::warn!(parent: &span, + context = "dkim", + event = "sign-failed", + reason = %err); + } } } } diff --git a/crates/smtp/src/scripts/exec.rs b/crates/smtp/src/scripts/exec.rs index 521620b5..11a8003f 100644 --- a/crates/smtp/src/scripts/exec.rs +++ b/crates/smtp/src/scripts/exec.rs @@ -23,11 +23,11 @@ use std::{sync::Arc, time::SystemTime}; +use common::listener::SessionStream; use mail_auth::common::resolver::ToReverseName; use sieve::{runtime::Variable, Envelope, Sieve}; use smtp_proto::*; use tokio::runtime::Handle; -use utils::listener::SessionStream; use crate::{core::Session, inbound::AuthResult}; diff --git a/crates/smtp/src/scripts/functions/array.rs b/crates/smtp/src/scripts/functions/array.rs deleted file mode 100644 index 450de724..00000000 --- a/crates/smtp/src/scripts/functions/array.rs +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::collections::{HashMap, HashSet}; - -use sieve::{runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -pub fn fn_count<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match &v[0] { - Variable::Array(a) => a.len(), - v => { - if !v.is_empty() { - 1 - } else { - 0 - } - } - } - .into() -} - -pub fn fn_sort<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let is_asc = v[1].to_bool(); - let mut arr = (*v[0].to_array()).clone(); - if is_asc { - arr.sort_unstable_by(|a, b| b.cmp(a)); - } else { - arr.sort_unstable(); - } - arr.into() -} - -pub fn fn_dedup<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let arr = v[0].to_array(); - let mut result = Vec::with_capacity(arr.len()); - - for item in arr.iter() { - if !result.contains(item) { - result.push(item.clone()); - } - } - - result.into() -} - -pub fn fn_cosine_similarity<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let mut word_freq: HashMap = HashMap::new(); - - for (idx, var) in v.into_iter().enumerate() { - match var { - Variable::Array(l) => { - for item in l.iter() { - word_freq.entry(item.clone()).or_insert([0, 0])[idx] += 1; - } - } - _ => { - for char in var.to_string().chars() { - word_freq.entry(char.to_string().into()).or_insert([0, 0])[idx] += 1; - } - } - } - } - - let mut dot_product = 0; - let mut magnitude_a = 0; - let mut magnitude_b = 0; - - for (_word, count) in word_freq.iter() { - dot_product += count[0] * count[1]; - magnitude_a += count[0] * count[0]; - magnitude_b += count[1] * count[1]; - } - - if magnitude_a != 0 && magnitude_b != 0 { - dot_product as f64 / (magnitude_a as f64).sqrt() / (magnitude_b as f64).sqrt() - } else { - 0.0 - } - .into() -} - -pub fn fn_jaccard_similarity<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let mut word_freq = [HashSet::new(), HashSet::new()]; - - for (idx, var) in v.into_iter().enumerate() { - match var { - Variable::Array(l) => { - for item in l.iter() { - word_freq[idx].insert(item.clone()); - } - } - _ => { - for char in var.to_string().chars() { - word_freq[idx].insert(char.to_string().into()); - } - } - } - } - - let intersection_size = word_freq[0].intersection(&word_freq[1]).count(); - let union_size = word_freq[0].union(&word_freq[1]).count(); - - if union_size != 0 { - intersection_size as f64 / union_size as f64 - } else { - 0.0 - } - .into() -} - -pub fn fn_is_intersect<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match (&v[0], &v[1]) { - (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), - (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), - _ => false, - } - .into() -} - -pub fn fn_winnow<'x>(_: &'x Context<'x, SieveContext>, mut v: Vec) -> Variable { - match v.remove(0) { - Variable::Array(a) => a - .iter() - .filter(|i| !i.is_empty()) - .cloned() - .collect::>() - .into(), - v => v, - } -} diff --git a/crates/smtp/src/scripts/functions/email.rs b/crates/smtp/src/scripts/functions/email.rs deleted file mode 100644 index 93bf4264..00000000 --- a/crates/smtp/src/scripts/functions/email.rs +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use sieve::{runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -use super::ApplyString; - -pub fn fn_is_email<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let mut last_ch = 0; - let mut in_quote = false; - let mut at_count = 0; - let mut dot_count = 0; - let mut lp_len = 0; - let mut value = 0; - - for ch in v[0].to_string().bytes() { - match ch { - b'0'..=b'9' - | b'a'..=b'z' - | b'A'..=b'Z' - | b'!' - | b'#' - | b'$' - | b'%' - | b'&' - | b'\'' - | b'*' - | b'+' - | b'-' - | b'/' - | b'=' - | b'?' - | b'^' - | b'_' - | b'`' - | b'{' - | b'|' - | b'}' - | b'~' - | 0x7f..=u8::MAX => { - value += 1; - } - b'.' if !in_quote => { - if last_ch != b'.' && last_ch != b'@' && value != 0 { - value += 1; - if at_count == 1 { - dot_count += 1; - } - } else { - return false.into(); - } - } - b'@' if !in_quote => { - at_count += 1; - lp_len = value; - value = 0; - } - b'>' | b':' | b',' | b' ' if in_quote => { - value += 1; - } - b'\"' if !in_quote || last_ch != b'\\' => { - in_quote = !in_quote; - } - b'\\' if in_quote && last_ch != b'\\' => (), - _ => { - if !in_quote { - return false.into(); - } - } - } - - last_ch = ch; - } - - (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() -} - -pub fn fn_email_part<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| { - s.rsplit_once('@') - .map(|(u, d)| match v[1].to_string().as_ref() { - "local" => Variable::from(u.trim()), - "domain" => Variable::from(d.trim()), - _ => Variable::default(), - }) - .unwrap_or_default() - }) -} - -#[derive(PartialEq, Eq, Clone, Copy)] -enum MatchPart { - Sld, - Tld, - Host, -} - -pub fn fn_domain_part<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let match_part = match v[1].to_string().as_ref() { - "sld" => MatchPart::Sld, - "tld" => MatchPart::Tld, - "host" => MatchPart::Host, - _ => return Variable::default(), - }; - - v[0].transform(|domain| { - let d = domain.trim().to_lowercase(); - let mut seen_dot = false; - for (pos, ch) in d.as_bytes().iter().enumerate().rev() { - if *ch == b'.' { - if seen_dot { - let maybe_domain = - std::str::from_utf8(&d.as_bytes()[pos + 1..]).unwrap_or_default(); - if !ctx.context().psl.contains(maybe_domain) { - return if match_part == MatchPart::Sld { - maybe_domain - } else { - std::str::from_utf8(&d.as_bytes()[..pos]).unwrap_or_default() - } - .to_string() - .into(); - } - } else if match_part == MatchPart::Tld { - return std::str::from_utf8(&d.as_bytes()[pos + 1..]) - .unwrap_or_default() - .to_string() - .into(); - } else { - seen_dot = true; - } - } - } - - if seen_dot { - if match_part == MatchPart::Sld { - d.into() - } else { - Variable::default() - } - } else if match_part == MatchPart::Host { - d.into() - } else { - Variable::default() - } - }) -} diff --git a/crates/smtp/src/scripts/functions/header.rs b/crates/smtp/src/scripts/functions/header.rs deleted file mode 100644 index 5ee773cc..00000000 --- a/crates/smtp/src/scripts/functions/header.rs +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue, MimeHeaders}; -use sieve::{compiler::ReceivedPart, runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -use super::ApplyString; - -pub fn fn_received_part<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - if let (Ok(part), Some(HeaderValue::Received(rcvd))) = ( - ReceivedPart::try_from(v[1].to_string().as_ref()), - ctx.message() - .part(ctx.part()) - .and_then(|p| { - p.headers - .iter() - .filter(|h| h.name == HeaderName::Received) - .nth((v[0].to_integer() as usize).saturating_sub(1)) - }) - .map(|h| &h.value), - ) { - part.eval(rcvd).unwrap_or_default() - } else { - Variable::default() - } -} - -pub fn fn_is_encoding_problem<'x>( - ctx: &'x Context<'x, SieveContext>, - _: Vec, -) -> Variable { - ctx.message() - .part(ctx.part()) - .map(|p| p.is_encoding_problem) - .unwrap_or_default() - .into() -} - -pub fn fn_is_attachment<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec) -> Variable { - ctx.message().attachments.contains(&ctx.part()).into() -} - -pub fn fn_is_body<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec) -> Variable { - (ctx.message().text_body.contains(&ctx.part()) || ctx.message().html_body.contains(&ctx.part())) - .into() -} - -pub fn fn_attachment_name<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec) -> Variable { - ctx.message() - .part(ctx.part()) - .and_then(|p| p.attachment_name()) - .unwrap_or_default() - .into() -} - -pub fn fn_mime_part_len<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec) -> Variable { - ctx.message() - .part(ctx.part()) - .map(|p| p.len()) - .unwrap_or_default() - .into() -} - -pub fn fn_thread_name<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| thread_name(s).into()) -} - -pub fn fn_is_header_utf8_valid<'x>( - ctx: &'x Context<'x, SieveContext>, - v: Vec, -) -> Variable { - ctx.message() - .part(ctx.part()) - .map(|p| { - let raw = ctx.message().raw_message(); - let mut is_valid = true; - if let Some(header_name) = HeaderName::parse(v[0].to_string().as_ref()) { - for header in &p.headers { - if header.name == header_name - && raw - .get(header.offset_start()..header.offset_end()) - .and_then(|raw| std::str::from_utf8(raw).ok()) - .is_none() - { - is_valid = false; - break; - } - } - } else { - is_valid = raw - .get(p.raw_header_offset()..p.raw_body_offset()) - .and_then(|raw| std::str::from_utf8(raw).ok()) - .is_some(); - } - - Variable::from(is_valid) - }) - .unwrap_or(Variable::Integer(1)) -} diff --git a/crates/smtp/src/scripts/functions/html.rs b/crates/smtp/src/scripts/functions/html.rs deleted file mode 100644 index 9879075f..00000000 --- a/crates/smtp/src/scripts/functions/html.rs +++ /dev/null @@ -1,441 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::borrow::Cow; - -use mail_parser::decoders::html::{add_html_token, html_to_text}; -use sieve::{runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -pub fn fn_html_to_text<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - html_to_text(v[0].to_string().as_ref()).into() -} - -pub fn fn_html_has_tag<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].as_array() - .map(|arr| { - let token = v[1].to_string(); - arr.iter().any(|v| { - v.to_string() - .as_ref() - .strip_prefix('<') - .map_or(false, |tag| tag.starts_with(token.as_ref())) - }) - }) - .unwrap_or_default() - .into() -} - -pub fn fn_html_attr_size<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let t = v[0].to_string(); - let mut dimension = None; - - if let Some(value) = get_attribute(t.as_ref(), v[1].to_string().as_ref()) { - let value = value.trim(); - if let Some(pct) = value.strip_suffix('%') { - if let Ok(pct) = pct.trim().parse::() { - dimension = ((v[2].to_integer() * pct as i64) / 100).into(); - } - } else if let Ok(value) = value.parse::() { - dimension = (value as i64).into(); - } - } - - dimension.map(Variable::Integer).unwrap_or_default() -} - -pub fn fn_html_attrs<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - html_attr_tokens( - v[0].to_string().as_ref(), - v[1].to_string().as_ref(), - v[2].to_string_array(), - ) - .into() -} - -pub fn fn_html_attr<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - get_attribute(v[0].to_string().as_ref(), v[1].to_string().as_ref()) - .map(Variable::from) - .unwrap_or_default() -} - -pub fn html_to_tokens(input: &str) -> Vec { - let input = input.as_bytes(); - let mut iter = input.iter().enumerate(); - let mut tags = vec![]; - - let mut is_token_start = true; - let mut is_after_space = false; - let mut is_new_line = true; - - let mut token_start = 0; - let mut token_end = 0; - - let mut text = String::from("_"); - - while let Some((pos, &ch)) = iter.next() { - match ch { - b'<' => { - if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space, - ); - is_after_space = false; - is_token_start = true; - } - if text.len() > 1 { - tags.push(Variable::String(text.into())); - text = String::from("_"); - } - - let mut tag = vec![b'<']; - if matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { - let mut last_ch: u8 = 0; - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if tag.len() > 3 - && matches!(tag.last(), Some(b'-')) - && matches!(tag.get(tag.len() - 2), Some(b'-')) => - { - break; - } - b' ' | b'\t' | b'\r' | b'\n' => { - if last_ch != b' ' { - tag.push(b' '); - } else { - last_ch = b' '; - } - continue; - } - _ => { - tag.push(ch); - } - } - last_ch = ch; - } - } else { - let mut in_quote = false; - let mut last_ch = b' '; - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if !in_quote => { - break; - } - b'"' => { - in_quote = !in_quote; - tag.push(b'"'); - } - b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { - if last_ch != b' ' { - tag.push(b' '); - last_ch = b' '; - } - continue; - } - b'/' if !in_quote => { - tag.push(b'/'); - last_ch = b' '; - continue; - } - _ => { - tag.push(if in_quote { - ch - } else { - ch.to_ascii_lowercase() - }); - } - } - last_ch = ch; - } - } - tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default().into(), - )); - continue; - } - b' ' | b'\t' | b'\r' | b'\n' => { - if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space && !is_new_line, - ); - is_new_line = false; - } - is_after_space = true; - is_token_start = true; - continue; - } - b'&' if !is_token_start => { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space && !is_new_line, - ); - is_new_line = false; - is_token_start = true; - is_after_space = false; - } - b';' if !is_token_start => { - add_html_token( - &mut text, - &input[token_start..pos + 1], - is_after_space && !is_new_line, - ); - is_token_start = true; - is_after_space = false; - is_new_line = false; - continue; - } - _ => (), - } - - if is_token_start { - token_start = pos; - is_token_start = false; - } - token_end = pos; - } - - if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space && !is_new_line, - ); - } - if text.len() > 1 { - tags.push(Variable::String(text.into())); - } - - tags -} - -pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec>) -> Vec { - let input = input.as_bytes(); - let mut iter = input.iter().enumerate().peekable(); - let mut tags = vec![]; - - while let Some((mut pos, &ch)) = iter.next() { - if ch == b'<' { - if !matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { - let mut in_quote = false; - let mut last_ch_pos: usize = 0; - - while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace()) { - pos += 1; - iter.next(); - } - - let found_tag = tag.is_empty() - || (matches!(input.get(pos + 1..pos + tag.len() + 1), Some(t) if t.eq_ignore_ascii_case(tag.as_bytes())) - && matches!(input.get(pos + tag.len() + 1), Some(ch) if ch.is_ascii_whitespace())); - - 'outer: while let Some((pos, &ch)) = iter.next() { - match ch { - b'>' if !in_quote => { - break; - } - b'"' => { - in_quote = !in_quote; - } - b'=' if found_tag - && !in_quote - && attrs.iter().any(|attr| matches!(input.get(last_ch_pos.saturating_sub(attr.len()) + 1..last_ch_pos + 1), Some(a) if a.eq_ignore_ascii_case(attr.as_bytes()))) - && matches!(input.get(last_ch_pos + 1), Some(ch) if ch.is_ascii_whitespace() || *ch == b'=') => - { - while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace()) - { - iter.next(); - } - let mut tag = vec![]; - - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if !in_quote => { - if !tag.is_empty() { - tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default().into(), - )); - } - break 'outer; - } - b'"' => { - if in_quote { - in_quote = false; - break; - } else { - in_quote = true; - } - } - b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { - break; - } - _ => { - tag.push(ch); - } - } - } - - if !tag.is_empty() { - tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default().into(), - )); - } - } - b' ' | b'\t' | b'\r' | b'\n' => {} - _ => { - last_ch_pos = pos; - } - } - } - } else { - let mut last_ch: u8 = 0; - let mut before_last_ch: u8 = 0; - - for (_, &ch) in iter.by_ref() { - if ch == b'>' && last_ch == b'-' && before_last_ch == b'-' { - break; - } - before_last_ch = last_ch; - last_ch = ch; - } - } - } - } - - tags -} - -pub fn html_img_area(arr: &[Variable]) -> u32 { - arr.iter() - .filter_map(|v| { - let t = v.to_string(); - if t.starts_with("() { - let size = if idx == 0 { 800 } else { 600 }; - dimensions[idx] = (size * pct) / 100; - } - } else if let Ok(value) = value.parse::() { - dimensions[idx] = value; - } - } - } - - Some(dimensions[0].saturating_mul(dimensions[1])) - } else { - None - } - }) - .sum::() -} - -pub fn get_attribute<'x>(tag: &'x str, attr_name: &str) -> Option<&'x str> { - let tag = tag.as_bytes(); - let attr_name = attr_name.as_bytes(); - let mut iter = tag.iter().enumerate().peekable(); - let mut in_quote = false; - let mut start_pos = usize::MAX; - let mut end_pos = usize::MAX; - - while let Some((pos, ch)) = iter.next() { - match ch { - b'=' if !in_quote => { - if start_pos != usize::MAX - && end_pos != usize::MAX - && tag - .get(start_pos..end_pos + 1) - .map_or(false, |name| name == attr_name) - { - let mut token_start = 0; - let mut token_end = 0; - - for (pos, ch) in iter.by_ref() { - match ch { - b'"' => { - if !in_quote { - token_start = pos + 1; - in_quote = true; - } else { - token_end = pos; - break; - } - } - b' ' if !in_quote => { - if token_start != 0 { - token_end = pos; - break; - } - } - _ => { - if token_start == 0 { - token_start = pos; - } - } - } - } - - return if token_start > 0 { - if token_end == 0 { - token_end = tag.len(); - } - Some(std::str::from_utf8(&tag[token_start..token_end]).unwrap_or_default()) - } else { - None - }; - } else { - start_pos = usize::MAX; - end_pos = usize::MAX; - } - } - b'"' => { - in_quote = !in_quote; - } - b' ' => { - if !in_quote && !matches!(iter.peek(), Some((_, b'='))) { - start_pos = usize::MAX; - end_pos = usize::MAX; - } - } - _ => { - if !in_quote { - if start_pos == usize::MAX { - start_pos = pos; - } - end_pos = pos; - } - } - } - } - - None -} diff --git a/crates/smtp/src/scripts/functions/image.rs b/crates/smtp/src/scripts/functions/image.rs deleted file mode 100644 index ffba52bd..00000000 --- a/crates/smtp/src/scripts/functions/image.rs +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use sieve::{runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -pub fn fn_img_metadata<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - ctx.message() - .part(ctx.part()) - .map(|p| p.contents()) - .and_then(|bytes| { - let arg = v[1].to_string(); - match arg.as_ref() { - "type" => imagesize::image_type(bytes).ok().map(|t| { - Variable::from(match t { - imagesize::ImageType::Aseprite => "aseprite", - imagesize::ImageType::Avif => "avif", - imagesize::ImageType::Bmp => "bmp", - imagesize::ImageType::Dds => "dds", - imagesize::ImageType::Exr => "exr", - imagesize::ImageType::Farbfeld => "farbfeld", - imagesize::ImageType::Gif => "gif", - imagesize::ImageType::Hdr => "hdr", - imagesize::ImageType::Heif => "heif", - imagesize::ImageType::Ico => "ico", - imagesize::ImageType::Jpeg => "jpeg", - imagesize::ImageType::Jxl => "jxl", - imagesize::ImageType::Ktx2 => "ktx2", - imagesize::ImageType::Png => "png", - imagesize::ImageType::Pnm => "pnm", - imagesize::ImageType::Psd => "psd", - imagesize::ImageType::Qoi => "qoi", - imagesize::ImageType::Tga => "tga", - imagesize::ImageType::Tiff => "tiff", - imagesize::ImageType::Vtf => "vtf", - imagesize::ImageType::Webp => "webp", - }) - }), - "width" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.width as i64)), - "height" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.height as i64)), - "area" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.width.saturating_mul(s.height) as i64)), - "dimension" => imagesize::blob_size(bytes) - .ok() - .map(|s| Variable::Integer(s.width.saturating_add(s.height) as i64)), - _ => None, - } - }) - .unwrap_or_default() -} diff --git a/crates/smtp/src/scripts/functions/misc.rs b/crates/smtp/src/scripts/functions/misc.rs deleted file mode 100644 index d7bc229a..00000000 --- a/crates/smtp/src/scripts/functions/misc.rs +++ /dev/null @@ -1,123 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::net::IpAddr; - -use mail_auth::common::resolver::ToReverseName; -use sha1::Sha1; -use sha2::{Sha256, Sha512}; -use sieve::{runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -use super::ApplyString; - -pub fn fn_is_empty<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.is_empty(), - Variable::Integer(_) | Variable::Float(_) => false, - Variable::Array(a) => a.is_empty(), - } - .into() -} - -pub fn fn_is_number<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into() -} - -pub fn fn_is_ip_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string().parse::().is_ok().into() -} - -pub fn fn_is_ipv4_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .parse::() - .map_or(false, |ip| matches!(ip, IpAddr::V4(_))) - .into() -} - -pub fn fn_is_ipv6_addr<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .parse::() - .map_or(false, |ip| matches!(ip, IpAddr::V6(_))) - .into() -} - -pub fn fn_ip_reverse_name<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .parse::() - .map(|ip| ip.to_reverse_name()) - .unwrap_or_default() - .into() -} - -pub fn fn_detect_file_type<'x>(ctx: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - ctx.message() - .part(ctx.part()) - .and_then(|p| infer::get(p.contents())) - .map(|t| { - Variable::from( - if v[0].to_string() != "ext" { - t.mime_type() - } else { - t.extension() - } - .to_string(), - ) - }) - .unwrap_or_default() -} - -pub fn fn_hash<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - use sha1::Digest; - let hash = v[1].to_string(); - - v[0].transform(|value| match hash.as_ref() { - "md5" => format!("{:x}", md5::compute(value.as_bytes())).into(), - "sha1" => { - let mut hasher = Sha1::new(); - hasher.update(value.as_bytes()); - format!("{:x}", hasher.finalize()).into() - } - "sha256" => { - let mut hasher = Sha256::new(); - hasher.update(value.as_bytes()); - format!("{:x}", hasher.finalize()).into() - } - "sha512" => { - let mut hasher = Sha512::new(); - hasher.update(value.as_bytes()); - format!("{:x}", hasher.finalize()).into() - } - _ => Variable::default(), - }) -} - -pub fn fn_is_var_names<'x>(ctx: &'x Context<'x, SieveContext>, _: Vec) -> Variable { - Variable::Array( - ctx.global_variable_names() - .map(|v| Variable::from(v.to_uppercase())) - .collect::>() - .into(), - ) -} diff --git a/crates/smtp/src/scripts/functions/mod.rs b/crates/smtp/src/scripts/functions/mod.rs deleted file mode 100644 index 5a00ad43..00000000 --- a/crates/smtp/src/scripts/functions/mod.rs +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -mod array; -mod email; -mod header; -pub mod html; -mod image; -mod misc; -mod text; -mod unicode; -mod url; - -use sieve::{runtime::Variable, FunctionMap}; - -use crate::config::scripts::SieveContext; - -use self::{ - array::*, email::*, header::*, html::*, image::*, misc::*, text::*, unicode::*, url::*, -}; - -pub fn register_functions() -> FunctionMap { - FunctionMap::new() - .with_function("trim", fn_trim) - .with_function("trim_start", fn_trim_start) - .with_function("trim_end", fn_trim_end) - .with_function("len", fn_len) - .with_function("count", fn_count) - .with_function("is_empty", fn_is_empty) - .with_function("is_number", fn_is_number) - .with_function("is_ascii", fn_is_ascii) - .with_function("to_lowercase", fn_to_lowercase) - .with_function("to_uppercase", fn_to_uppercase) - .with_function("detect_language", fn_detect_language) - .with_function("is_email", fn_is_email) - .with_function("thread_name", fn_thread_name) - .with_function("html_to_text", fn_html_to_text) - .with_function("is_uppercase", fn_is_uppercase) - .with_function("is_lowercase", fn_is_lowercase) - .with_function("has_digits", fn_has_digits) - .with_function("count_spaces", fn_count_spaces) - .with_function("count_uppercase", fn_count_uppercase) - .with_function("count_lowercase", fn_count_lowercase) - .with_function("count_chars", fn_count_chars) - .with_function("dedup", fn_dedup) - .with_function("lines", fn_lines) - .with_function("is_header_utf8_valid", fn_is_header_utf8_valid) - .with_function("img_metadata", fn_img_metadata) - .with_function("is_ip_addr", fn_is_ip_addr) - .with_function("is_ipv4_addr", fn_is_ipv4_addr) - .with_function("is_ipv6_addr", fn_is_ipv6_addr) - .with_function("ip_reverse_name", fn_ip_reverse_name) - .with_function("winnow", fn_winnow) - .with_function("has_zwsp", fn_has_zwsp) - .with_function("has_obscured", fn_has_obscured) - .with_function("is_single_script", fn_is_single_script) - .with_function("puny_decode", fn_puny_decode) - .with_function("unicode_skeleton", fn_unicode_skeleton) - .with_function("cure_text", fn_cure_text) - .with_function("detect_file_type", fn_detect_file_type) - .with_function_args("sort", fn_sort, 2) - .with_function_args("tokenize", fn_tokenize, 2) - .with_function_args("email_part", fn_email_part, 2) - .with_function_args("domain_part", fn_domain_part, 2) - .with_function_args("eq_ignore_case", fn_eq_ignore_case, 2) - .with_function_args("contains", fn_contains, 2) - .with_function_args("contains_ignore_case", fn_contains_ignore_case, 2) - .with_function_args("starts_with", fn_starts_with, 2) - .with_function_args("ends_with", fn_ends_with, 2) - .with_function_args("received_part", fn_received_part, 2) - .with_function_args("cosine_similarity", fn_cosine_similarity, 2) - .with_function_args("jaccard_similarity", fn_jaccard_similarity, 2) - .with_function_args("levenshtein_distance", fn_levenshtein_distance, 2) - .with_function_args("html_has_tag", fn_html_has_tag, 2) - .with_function_args("html_attr", fn_html_attr, 2) - .with_function_args("html_attrs", fn_html_attrs, 3) - .with_function_args("html_attr_size", fn_html_attr_size, 3) - .with_function_args("uri_part", fn_uri_part, 2) - .with_function_args("substring", fn_substring, 3) - .with_function_args("split", fn_split, 2) - .with_function_args("rsplit", fn_rsplit, 2) - .with_function_args("split_once", fn_split_once, 2) - .with_function_args("rsplit_once", fn_rsplit_once, 2) - .with_function_args("strip_prefix", fn_strip_prefix, 2) - .with_function_args("strip_suffix", fn_strip_suffix, 2) - .with_function_args("is_intersect", fn_is_intersect, 2) - .with_function_args("hash", fn_hash, 2) - .with_function_no_args("is_encoding_problem", fn_is_encoding_problem) - .with_function_no_args("is_attachment", fn_is_attachment) - .with_function_no_args("is_body", fn_is_body) - .with_function_no_args("var_names", fn_is_var_names) - .with_function_no_args("attachment_name", fn_attachment_name) - .with_function_no_args("mime_part_len", fn_mime_part_len) -} - -pub trait ApplyString<'x> { - fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable; -} - -impl<'x> ApplyString<'x> for Variable { - fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable { - match self { - Variable::String(s) => f(s), - Variable::Array(list) => list - .iter() - .map(|v| match v { - Variable::String(s) => f(s), - v => f(v.to_string().as_ref()), - }) - .collect::>() - .into(), - v => f(v.to_string().as_ref()), - } - } -} diff --git a/crates/smtp/src/scripts/functions/text.rs b/crates/smtp/src/scripts/functions/text.rs deleted file mode 100644 index 567a43cf..00000000 --- a/crates/smtp/src/scripts/functions/text.rs +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use nlp::tokenizers::types::{TokenType, TypesTokenizer}; -use sieve::{runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -use super::{html::html_to_tokens, ApplyString}; - -pub fn fn_trim<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.trim())) -} - -pub fn fn_trim_end<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.trim_end())) -} - -pub fn fn_trim_start<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.trim_start())) -} - -pub fn fn_len<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.len(), - Variable::Array(a) => a.len(), - v => v.to_string().len(), - } - .into() -} - -pub fn fn_to_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.to_lowercase())) -} - -pub fn fn_to_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| Variable::from(s.to_uppercase())) -} - -pub fn fn_is_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| { - s.chars() - .filter(|c| c.is_alphabetic()) - .all(|c| c.is_uppercase()) - .into() - }) -} - -pub fn fn_is_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| { - s.chars() - .filter(|c| c.is_alphabetic()) - .all(|c| c.is_lowercase()) - .into() - }) -} - -pub fn fn_has_digits<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|s| s.chars().any(|c| c.is_ascii_digit()).into()) -} - -pub fn tokenize_words(v: &Variable) -> Variable { - v.to_string() - .split_whitespace() - .filter(|word| word.chars().all(|c| c.is_alphanumeric())) - .map(|word| Variable::from(word.to_string())) - .collect::>() - .into() -} - -pub fn fn_tokenize<'x>(ctx: &'x Context<'x, SieveContext>, mut v: Vec) -> Variable { - let (urls, urls_without_scheme, emails) = match v[1].to_string().as_ref() { - "html" => return html_to_tokens(v[0].to_string().as_ref()).into(), - "words" => return tokenize_words(&v[0]), - "uri" | "url" => (true, true, true), - "uri_strict" | "url_strict" => (true, false, false), - "email" => (false, false, true), - _ => return Variable::default(), - }; - - match v.remove(0) { - v @ (Variable::String(_) | Variable::Array(_)) => { - TypesTokenizer::new(v.to_string().as_ref(), &ctx.context().psl) - .tokenize_numbers(false) - .tokenize_urls(urls) - .tokenize_urls_without_scheme(urls_without_scheme) - .tokenize_emails(emails) - .filter_map(|t| match t.word { - TokenType::Url(text) if urls => Variable::from(text.to_string()).into(), - TokenType::UrlNoScheme(text) if urls_without_scheme => { - Variable::from(format!("https://{text}")).into() - } - TokenType::Email(text) if emails => Variable::from(text.to_string()).into(), - _ => None, - }) - .collect::>() - .into() - } - v => v, - } -} - -pub fn fn_count_spaces<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .as_ref() - .chars() - .filter(|c| c.is_whitespace()) - .count() - .into() -} - -pub fn fn_count_uppercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .as_ref() - .chars() - .filter(|c| c.is_alphabetic() && c.is_uppercase()) - .count() - .into() -} - -pub fn fn_count_lowercase<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .as_ref() - .chars() - .filter(|c| c.is_alphabetic() && c.is_lowercase()) - .count() - .into() -} - -pub fn fn_count_chars<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string().as_ref().chars().count().into() -} - -pub fn fn_eq_ignore_case<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .eq_ignore_ascii_case(v[1].to_string().as_ref()) - .into() -} - -pub fn fn_contains<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.contains(v[1].to_string().as_ref()), - Variable::Array(arr) => arr.contains(&v[1]), - val => val.to_string().contains(v[1].to_string().as_ref()), - } - .into() -} - -pub fn fn_contains_ignore_case<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let needle = v[1].to_string(); - match &v[0] { - Variable::String(s) => s.to_lowercase().contains(&needle.to_lowercase()), - Variable::Array(arr) => arr.iter().any(|v| match v { - Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()), - _ => false, - }), - val => val.to_string().contains(needle.as_ref()), - } - .into() -} - -pub fn fn_starts_with<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .starts_with(v[1].to_string().as_ref()) - .into() -} - -pub fn fn_ends_with<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string().ends_with(v[1].to_string().as_ref()).into() -} - -pub fn fn_lines<'x>(_: &'x Context<'x, SieveContext>, mut v: Vec) -> Variable { - match v.remove(0) { - Variable::String(s) => s - .lines() - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into(), - val => val, - } -} - -pub fn fn_substring<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .chars() - .skip(v[1].to_usize()) - .take(v[2].to_usize()) - .collect::() - .into() -} - -pub fn fn_strip_prefix<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let prefix = v[1].to_string(); - v[0].transform(|s| { - s.strip_prefix(prefix.as_ref()) - .map(Variable::from) - .unwrap_or_default() - }) -} - -pub fn fn_strip_suffix<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let suffix = v[1].to_string(); - v[0].transform(|s| { - s.strip_suffix(suffix.as_ref()) - .map(Variable::from) - .unwrap_or_default() - }) -} - -pub fn fn_split<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .split(v[1].to_string().as_ref()) - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into() -} - -pub fn fn_rsplit<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .rsplit(v[1].to_string().as_ref()) - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into() -} - -pub fn fn_split_once<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .split_once(v[1].to_string().as_ref()) - .map(|(a, b)| { - Variable::Array( - vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), - ) - }) - .unwrap_or_default() -} - -pub fn fn_rsplit_once<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].to_string() - .rsplit_once(v[1].to_string().as_ref()) - .map(|(a, b)| { - Variable::Array( - vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), - ) - }) - .unwrap_or_default() -} - -/** - * `levenshtein-rs` - levenshtein - * - * MIT licensed. - * - * Copyright (c) 2016 Titus Wormer - */ -pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let a = v[0].to_string(); - let b = v[1].to_string(); - - let mut result = 0; - - /* Shortcut optimizations / degenerate cases. */ - if a == b { - return result.into(); - } - - let length_a = a.chars().count(); - let length_b = b.chars().count(); - - if length_a == 0 { - return length_b.into(); - } else if length_b == 0 { - return length_a.into(); - } - - /* Initialize the vector. - * - * This is why it’s fast, normally a matrix is used, - * here we use a single vector. */ - let mut cache: Vec = (1..).take(length_a).collect(); - let mut distance_a; - let mut distance_b; - - /* Loop. */ - for (index_b, code_b) in b.chars().enumerate() { - result = index_b; - distance_a = index_b; - - for (index_a, code_a) in a.chars().enumerate() { - distance_b = if code_a == code_b { - distance_a - } else { - distance_a + 1 - }; - - distance_a = cache[index_a]; - - result = if distance_a > result { - if distance_b > result { - result + 1 - } else { - distance_b - } - } else if distance_b > distance_a { - distance_a + 1 - } else { - distance_b - }; - - cache[index_a] = result; - } - } - - result.into() -} - -pub fn fn_detect_language<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - whatlang::detect_lang(v[0].to_string().as_ref()) - .map(|l| l.code()) - .unwrap_or("unknown") - .into() -} diff --git a/crates/smtp/src/scripts/functions/unicode.rs b/crates/smtp/src/scripts/functions/unicode.rs deleted file mode 100644 index 2829cddd..00000000 --- a/crates/smtp/src/scripts/functions/unicode.rs +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use sieve::{runtime::Variable, Context}; -use unicode_security::MixedScript; - -use crate::config::scripts::SieveContext; - -pub fn fn_is_ascii<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.chars().all(|c| c.is_ascii()), - Variable::Integer(_) | Variable::Float(_) => true, - Variable::Array(a) => a.iter().all(|v| match v { - Variable::String(s) => s.chars().all(|c| c.is_ascii()), - _ => true, - }), - } - .into() -} - -pub fn fn_has_zwsp<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.chars().any(|c| c.is_zwsp()), - Variable::Array(a) => a.iter().any(|v| match v { - Variable::String(s) => s.chars().any(|c| c.is_zwsp()), - _ => true, - }), - Variable::Integer(_) | Variable::Float(_) => false, - } - .into() -} - -pub fn fn_has_obscured<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.chars().any(|c| c.is_obscured()), - Variable::Array(a) => a.iter().any(|v| match v { - Variable::String(s) => s.chars().any(|c| c.is_obscured()), - _ => true, - }), - Variable::Integer(_) | Variable::Float(_) => false, - } - .into() -} - -trait CharUtils { - fn is_zwsp(&self) -> bool; - fn is_obscured(&self) -> bool; -} - -impl CharUtils for char { - fn is_zwsp(&self) -> bool { - matches!( - self, - '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{FEFF}' | '\u{00AD}' - ) - } - - fn is_obscured(&self) -> bool { - matches!( - self, - '\u{200B}'..='\u{200F}' - | '\u{2028}'..='\u{202F}' - | '\u{205F}'..='\u{206F}' - | '\u{FEFF}' - ) - } -} - -pub fn fn_cure_text<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - decancer::cure(v[0].to_string().as_ref(), decancer::Options::default()) - .map(|s| s.into_str()) - .unwrap_or_default() - .into() -} - -pub fn fn_unicode_skeleton<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - unicode_security::skeleton(v[0].to_string().as_ref()) - .collect::() - .into() -} - -pub fn fn_is_single_script<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let text = v[0].to_string(); - if !text.is_empty() { - text.as_ref().is_single_script() - } else { - true - } - .into() -} diff --git a/crates/smtp/src/scripts/functions/url.rs b/crates/smtp/src/scripts/functions/url.rs deleted file mode 100644 index b1f9ef3b..00000000 --- a/crates/smtp/src/scripts/functions/url.rs +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use hyper::Uri; -use sieve::{runtime::Variable, Context}; - -use crate::config::scripts::SieveContext; - -use super::ApplyString; - -pub fn fn_uri_part<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - let part = v[1].to_string(); - v[0].transform(|uri| { - uri.parse::() - .ok() - .and_then(|uri| match part.as_ref() { - "scheme" => uri.scheme_str().map(|s| Variable::from(s.to_string())), - "host" => uri.host().map(|s| Variable::from(s.to_string())), - "scheme_host" => uri - .scheme_str() - .and_then(|s| (s, uri.host()?).into()) - .map(|(s, h)| Variable::from(format!("{}://{}", s, h))), - "path" => Variable::from(uri.path().to_string()).into(), - "port" => uri.port_u16().map(|port| Variable::Integer(port as i64)), - "query" => uri.query().map(|s| Variable::from(s.to_string())), - "path_query" => uri.path_and_query().map(|s| Variable::from(s.to_string())), - "authority" => uri.authority().map(|s| Variable::from(s.to_string())), - _ => None, - }) - .unwrap_or_default() - }) -} - -pub fn fn_puny_decode<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - v[0].transform(|domain| { - if domain.contains("xn--") { - let mut decoded = String::with_capacity(domain.len()); - for part in domain.split('.') { - if !decoded.is_empty() { - decoded.push('.'); - } - - if let Some(puny) = part - .strip_prefix("xn--") - .and_then(idna::punycode::decode_to_string) - { - decoded.push_str(&puny); - } else { - decoded.push_str(part); - } - } - decoded.into() - } else { - domain.into() - } - }) -} diff --git a/crates/smtp/src/scripts/mod.rs b/crates/smtp/src/scripts/mod.rs index e461f1f8..e8e5ab94 100644 --- a/crates/smtp/src/scripts/mod.rs +++ b/crates/smtp/src/scripts/mod.rs @@ -24,13 +24,12 @@ use std::{borrow::Cow, sync::Arc}; use ahash::AHashMap; +use common::scripts::ScriptModification; use sieve::{runtime::Variable, Envelope}; pub mod envelope; pub mod event_loop; pub mod exec; -pub mod functions; -pub mod plugins; #[derive(Debug)] pub enum ScriptResult { @@ -45,18 +44,6 @@ pub enum ScriptResult { Discard, } -#[derive(Debug)] -pub enum ScriptModification { - SetEnvelope { - name: Envelope, - value: String, - }, - AddHeader { - name: Arc, - value: Arc, - }, -} - pub struct ScriptParameters { message: Option>>, variables: AHashMap, Variable>, diff --git a/crates/smtp/src/scripts/plugins/bayes.rs b/crates/smtp/src/scripts/plugins/bayes.rs deleted file mode 100644 index 9e8df73d..00000000 --- a/crates/smtp/src/scripts/plugins/bayes.rs +++ /dev/null @@ -1,362 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use nlp::{ - bayes::{ - cache::BayesTokenCache, tokenize::BayesTokenizer, BayesClassifier, BayesModel, TokenHash, - Weights, - }, - tokenizers::osb::{OsbToken, OsbTokenizer}, -}; -use sieve::{runtime::Variable, FunctionMap}; -use store::{write::key::KeySerializer, LookupStore, U64_LEN}; -use tokio::runtime::Handle; - -use crate::config::scripts::SieveContext; - -use super::PluginContext; - -pub fn register_train(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("bayes_train", plugin_id, 3); -} - -pub fn register_untrain(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("bayes_untrain", plugin_id, 3); -} - -pub fn register_classify(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("bayes_classify", plugin_id, 3); -} - -pub fn register_is_balanced(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("bayes_is_balanced", plugin_id, 3); -} - -pub fn exec_train(ctx: PluginContext<'_>) -> Variable { - train(ctx, true) -} - -pub fn exec_untrain(ctx: PluginContext<'_>) -> Variable { - train(ctx, false) -} - -fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { - let span: &tracing::Span = ctx.span; - let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.lookup_stores.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_lookup_store), - }; - - let store = if let Some(store) = store { - store - } else { - tracing::warn!( - parent: span, - context = "sieve:bayes_train", - event = "failed", - reason = "Unknown store id", - lookup_store = ctx.arguments[0].to_string().as_ref(), - ); - return false.into(); - }; - let text = ctx.arguments[1].to_string(); - let is_spam = ctx.arguments[2].to_bool(); - if text.is_empty() { - return false.into(); - } - let handle = ctx.handle; - let ctx = ctx.core.sieve.runtime.context(); - - // Train the model - let mut model = BayesModel::default(); - model.train( - OsbTokenizer::new(BayesTokenizer::new(text.as_ref(), &ctx.psl), 5), - is_spam, - ); - if model.weights.is_empty() { - return false.into(); - } - - tracing::debug!( - parent: span, - context = "sieve:bayes_train", - event = "train", - is_spam = is_spam, - num_tokens = model.weights.len(), - ); - - // Update weight and invalidate cache - if is_train { - for (hash, weights) in model.weights { - if handle - .block_on( - store.counter_incr( - KeySerializer::new(U64_LEN) - .write(hash.h1) - .write(hash.h2) - .finalize(), - weights.into(), - None, - false, - ), - ) - .is_err() - { - return false.into(); - } - ctx.bayes_cache.invalidate(&hash); - } - - // Update training counts - let weights = if is_spam { - Weights { spam: 1, ham: 0 } - } else { - Weights { spam: 0, ham: 1 } - }; - if handle - .block_on( - store.counter_incr( - KeySerializer::new(U64_LEN) - .write(0u64) - .write(0u64) - .finalize(), - weights.into(), - None, - false, - ), - ) - .is_err() - { - return false.into(); - } - } else { - //TODO: Implement untrain - return false.into(); - } - - ctx.bayes_cache.invalidate(&TokenHash::default()); - - true.into() -} - -pub fn exec_classify(ctx: PluginContext<'_>) -> Variable { - let span = ctx.span; - let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.lookup_stores.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_lookup_store), - }; - let store = if let Some(store) = store { - store - } else { - tracing::warn!( - parent: span, - context = "sieve:bayes_classify", - event = "failed", - reason = "Unknown store id", - lookup_id = ctx.arguments[0].to_string().as_ref(), - ); - return Variable::default(); - }; - let text = ctx.arguments[1].to_string(); - if text.is_empty() { - return Variable::default(); - } - - // Create classifier from defaults - let mut classifier = BayesClassifier::default(); - if let Some(params) = ctx.arguments[2].as_array() { - if let Some(Variable::Integer(value)) = params.first() { - classifier.min_token_hits = *value as u32; - } - if let Some(Variable::Integer(value)) = params.get(1) { - classifier.min_tokens = *value as u32; - } - if let Some(Variable::Float(value)) = params.get(2) { - classifier.min_prob_strength = *value; - } - if let Some(Variable::Integer(value)) = params.get(3) { - classifier.min_learns = *value as u32; - } - } - - let handle = ctx.handle; - let ctx = ctx.core.sieve.runtime.context(); - - // Obtain training counts - let (spam_learns, ham_learns) = if let Some(weights) = - ctx.bayes_cache - .get_or_update(TokenHash::default(), handle, store) - { - (weights.spam, weights.ham) - } else { - tracing::warn!( - parent: span, - context = "sieve:classify", - event = "failed", - reason = "Failed to obtain training counts", - ); - return Variable::default(); - }; - - // Make sure we have enough training data - if spam_learns < classifier.min_learns || ham_learns < classifier.min_learns { - tracing::debug!( - parent: span, - context = "sieve:bayes_classify", - event = "skip-classify", - reason = "Not enough training data", - spam_learns = %spam_learns, - ham_learns = %ham_learns); - return Variable::default(); - } - - // Classify the text - classifier - .classify( - OsbTokenizer::<_, TokenHash>::new(BayesTokenizer::new(text.as_ref(), &ctx.psl), 5) - .filter_map(|t| { - OsbToken { - inner: ctx.bayes_cache.get_or_update(t.inner, handle, store)?, - idx: t.idx, - } - .into() - }), - ham_learns, - spam_learns, - ) - .map(Variable::from) - .unwrap_or_default() -} - -pub fn exec_is_balanced(ctx: PluginContext<'_>) -> Variable { - let min_balance = match &ctx.arguments[2] { - Variable::Float(n) => *n, - Variable::Integer(n) => *n as f64, - _ => 0.0, - }; - - if min_balance == 0.0 { - return true.into(); - } - - let span = ctx.span; - let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.lookup_stores.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_lookup_store), - }; - let store = if let Some(store) = store { - store - } else { - tracing::warn!( - parent: span, - context = "sieve:bayes_is_balanced", - event = "failed", - reason = "Unknown store id", - lookup_id = ctx.arguments[0].to_string().as_ref(), - ); - return Variable::default(); - }; - let learn_spam = ctx.arguments[1].to_bool(); - - // Obtain training counts - let handle = ctx.handle; - let ctx = ctx.core.sieve.runtime.context(); - let (spam_learns, ham_learns) = if let Some(weights) = - ctx.bayes_cache - .get_or_update(TokenHash::default(), handle, store) - { - (weights.spam as f64, weights.ham as f64) - } else { - tracing::warn!( - parent: span, - context = "sieve:bayes_is_balanced", - event = "failed", - reason = "Failed to obtain training counts", - ); - return Variable::default(); - }; - - let result = if spam_learns > 0.0 || ham_learns > 0.0 { - if learn_spam { - (spam_learns / (ham_learns + 1.0)) <= 1.0 / min_balance - } else { - (ham_learns / (spam_learns + 1.0)) <= 1.0 / min_balance - } - } else { - true - }; - - tracing::debug!( - parent: span, - context = "sieve:bayes_is_balanced", - event = "result", - is_balanced = %result, - learn_spam = %learn_spam, - min_balance = %min_balance, - spam_learns = %spam_learns, - ham_learns = %ham_learns); - - result.into() -} - -trait LookupOrInsert { - fn get_or_update( - &self, - hash: TokenHash, - handle: &Handle, - get_token: &LookupStore, - ) -> Option; -} - -impl LookupOrInsert for BayesTokenCache { - fn get_or_update( - &self, - hash: TokenHash, - handle: &Handle, - get_token: &LookupStore, - ) -> Option { - if let Some(weights) = self.get(&hash) { - weights.unwrap_or_default().into() - } else if let Ok(num) = handle.block_on( - get_token.counter_get( - KeySerializer::new(U64_LEN) - .write(hash.h1) - .write(hash.h2) - .finalize(), - ), - ) { - if num != 0 { - let weights = Weights::from(num); - self.insert_positive(hash, weights); - weights - } else { - self.insert_negative(hash); - Weights::default() - } - .into() - } else { - // Something went wrong - None - } - } -} diff --git a/crates/smtp/src/scripts/plugins/dns.rs b/crates/smtp/src/scripts/plugins/dns.rs deleted file mode 100644 index a7fc7e30..00000000 --- a/crates/smtp/src/scripts/plugins/dns.rs +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::net::IpAddr; - -use mail_auth::{Error, IpLookupStrategy}; -use sieve::{runtime::Variable, FunctionMap}; - -use crate::config::scripts::SieveContext; - -use super::PluginContext; - -pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("dns_query", plugin_id, 2); -} - -pub fn register_exists(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("dns_exists", plugin_id, 2); -} - -pub fn exec(ctx: PluginContext<'_>) -> Variable { - let entry = ctx.arguments[0].to_string(); - let record_type = ctx.arguments[1].to_string(); - - if record_type.eq_ignore_ascii_case("ip") { - match ctx.handle.block_on(ctx.core.resolvers.dns.ip_lookup( - entry.as_ref(), - IpLookupStrategy::Ipv4thenIpv6, - 10, - )) { - Ok(result) => result - .iter() - .map(|ip| Variable::from(ip.to_string())) - .collect::>() - .into(), - Err(err) => err.short_error().into(), - } - } else if record_type.eq_ignore_ascii_case("mx") { - match ctx - .handle - .block_on(ctx.core.resolvers.dns.mx_lookup(entry.as_ref())) - { - Ok(result) => result - .iter() - .flat_map(|mx| { - mx.exchanges - .iter() - .map(|host| Variable::from(format!("{} {}", mx.preference, host))) - }) - .collect::>() - .into(), - Err(err) => err.short_error().into(), - } - } else if record_type.eq_ignore_ascii_case("txt") { - #[cfg(feature = "test_mode")] - { - if entry.contains("origin") { - return Variable::from("23028|US|arin|2002-01-04".to_string()); - } - } - - match ctx - .handle - .block_on(ctx.core.resolvers.dns.txt_raw_lookup(entry.as_ref())) - { - Ok(result) => Variable::from(String::from_utf8(result).unwrap_or_default()), - Err(err) => err.short_error().into(), - } - } else if record_type.eq_ignore_ascii_case("ptr") { - if let Ok(addr) = entry.parse::() { - match ctx.handle.block_on(ctx.core.resolvers.dns.ptr_lookup(addr)) { - Ok(result) => result - .iter() - .map(|host| Variable::from(host.to_string())) - .collect::>() - .into(), - Err(err) => err.short_error().into(), - } - } else { - Variable::default() - } - } else if record_type.eq_ignore_ascii_case("ipv4") { - #[cfg(feature = "test_mode")] - { - if entry.contains(".168.192.") { - let parts = entry.split('.').collect::>(); - return vec![Variable::from(format!("127.0.{}.{}", parts[1], parts[0]))].into(); - } - } - - match ctx - .handle - .block_on(ctx.core.resolvers.dns.ipv4_lookup(entry.as_ref())) - { - Ok(result) => result - .iter() - .map(|ip| Variable::from(ip.to_string())) - .collect::>() - .into(), - Err(err) => err.short_error().into(), - } - } else if record_type.eq_ignore_ascii_case("ipv6") { - match ctx - .handle - .block_on(ctx.core.resolvers.dns.ipv6_lookup(entry.as_ref())) - { - Ok(result) => result - .iter() - .map(|ip| Variable::from(ip.to_string())) - .collect::>() - .into(), - Err(err) => err.short_error().into(), - } - } else { - Variable::default() - } -} - -pub fn exec_exists(ctx: PluginContext<'_>) -> Variable { - let entry = ctx.arguments[0].to_string(); - let record_type = ctx.arguments[1].to_string(); - - if record_type.eq_ignore_ascii_case("ip") { - match ctx.handle.block_on(ctx.core.resolvers.dns.ip_lookup( - entry.as_ref(), - IpLookupStrategy::Ipv4thenIpv6, - 10, - )) { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } - } else if record_type.eq_ignore_ascii_case("mx") { - match ctx - .handle - .block_on(ctx.core.resolvers.dns.mx_lookup(entry.as_ref())) - { - Ok(result) => i64::from(result.iter().any(|mx| !mx.exchanges.is_empty())), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } - } else if record_type.eq_ignore_ascii_case("ptr") { - if let Ok(addr) = entry.parse::() { - match ctx.handle.block_on(ctx.core.resolvers.dns.ptr_lookup(addr)) { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } - } else { - -1 - } - } else if record_type.eq_ignore_ascii_case("ipv4") { - #[cfg(feature = "test_mode")] - { - if entry.starts_with("2.0.168.192.") { - return 1.into(); - } - } - - match ctx - .handle - .block_on(ctx.core.resolvers.dns.ipv4_lookup(entry.as_ref())) - { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } - } else if record_type.eq_ignore_ascii_case("ipv6") { - match ctx - .handle - .block_on(ctx.core.resolvers.dns.ipv6_lookup(entry.as_ref())) - { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } - } else { - -1 - } - .into() -} - -trait ShortError { - fn short_error(&self) -> &'static str; -} - -impl ShortError for mail_auth::Error { - fn short_error(&self) -> &'static str { - match self { - mail_auth::Error::DnsError(_) => "temp_fail", - mail_auth::Error::DnsRecordNotFound(_) => "not_found", - mail_auth::Error::Io(_) => "io_error", - mail_auth::Error::InvalidRecordType => "invalid_record", - _ => "unknown_error", - } - } -} diff --git a/crates/smtp/src/scripts/plugins/exec.rs b/crates/smtp/src/scripts/plugins/exec.rs deleted file mode 100644 index 053e7912..00000000 --- a/crates/smtp/src/scripts/plugins/exec.rs +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::process::Command; - -use sieve::{runtime::Variable, FunctionMap}; - -use crate::config::scripts::SieveContext; - -use super::PluginContext; - -pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("exec", plugin_id, 2); -} - -pub fn exec(ctx: PluginContext<'_>) -> Variable { - let span = ctx.span; - let mut arguments = ctx.arguments.into_iter(); - match Command::new( - arguments - .next() - .map(|a| a.to_string().into_owned()) - .unwrap_or_default(), - ) - .args( - arguments - .next() - .map(|a| a.into_string_array()) - .unwrap_or_default(), - ) - .output() - { - Ok(result) => result.status.success().into(), - Err(err) => { - tracing::warn!( - parent: span, - context = "sieve", - event = "execute-failed", - reason = %err, - ); - false.into() - } - } -} diff --git a/crates/smtp/src/scripts/plugins/headers.rs b/crates/smtp/src/scripts/plugins/headers.rs deleted file mode 100644 index 0160556c..00000000 --- a/crates/smtp/src/scripts/plugins/headers.rs +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use sieve::{runtime::Variable, FunctionMap}; - -use crate::{config::scripts::SieveContext, scripts::ScriptModification}; - -use super::PluginContext; - -pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("add_header", plugin_id, 2); -} - -pub fn exec(ctx: PluginContext<'_>) -> Variable { - if let (Variable::String(name), Variable::String(value)) = - (&ctx.arguments[0], &ctx.arguments[1]) - { - ctx.modifications.push(ScriptModification::AddHeader { - name: name.clone(), - value: value.clone(), - }); - true - } else { - false - } - .into() -} diff --git a/crates/smtp/src/scripts/plugins/http.rs b/crates/smtp/src/scripts/plugins/http.rs deleted file mode 100644 index 77dc4bad..00000000 --- a/crates/smtp/src/scripts/plugins/http.rs +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::time::Duration; - -use reqwest::redirect::Policy; -use sieve::{runtime::Variable, FunctionMap}; - -use crate::config::scripts::SieveContext; - -use super::PluginContext; - -pub fn register_header(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("http_header", plugin_id, 4); -} - -pub fn exec_header(ctx: PluginContext<'_>) -> Variable { - let url = ctx.arguments[0].to_string(); - let header = ctx.arguments[1].to_string(); - let agent = ctx.arguments[2].to_string(); - let timeout = ctx.arguments[3].to_string().parse::().unwrap_or(5000); - - #[cfg(feature = "test_mode")] - if url.contains("redirect.") { - return Variable::from(url.split_once("/?").unwrap().1.to_string()); - } - - if let Ok(client) = reqwest::Client::builder() - .user_agent(agent.as_ref()) - .timeout(Duration::from_millis(timeout)) - .redirect(Policy::none()) - .danger_accept_invalid_certs(true) - .build() - { - let _enter = ctx.handle.enter(); - ctx.handle - .block_on(client.get(url.as_ref()).send()) - .ok() - .and_then(|response| { - response - .headers() - .get(header.as_ref()) - .and_then(|h| h.to_str().ok()) - .map(|h| Variable::from(h.to_string())) - }) - .unwrap_or_default() - } else { - false.into() - } -} diff --git a/crates/smtp/src/scripts/plugins/lookup.rs b/crates/smtp/src/scripts/plugins/lookup.rs deleted file mode 100644 index 913a761b..00000000 --- a/crates/smtp/src/scripts/plugins/lookup.rs +++ /dev/null @@ -1,446 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - collections::HashSet, - io::{BufRead, BufReader}, - time::{Duration, Instant}, -}; - -use mail_auth::flate2; -use sieve::{runtime::Variable, FunctionMap}; -use store::{Deserialize, Value}; - -use crate::{ - config::scripts::{RemoteList, SieveContext}, - core::into_sieve_value, - USER_AGENT, -}; - -use super::PluginContext; - -pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("key_exists", plugin_id, 2); -} - -pub fn register_get(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("key_get", plugin_id, 2); -} - -pub fn register_set(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("key_set", plugin_id, 4); -} - -pub fn register_remote(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("key_exists_http", plugin_id, 3); -} - -pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("is_local_domain", plugin_id, 2); -} - -pub fn exec(ctx: PluginContext<'_>) -> Variable { - let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.lookup_stores.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_lookup_store), - }; - - if let Some(store) = store { - match &ctx.arguments[1] { - Variable::Array(items) => { - for item in items.iter() { - if !item.is_empty() - && ctx - .handle - .block_on(store.key_exists(item.to_string().into_owned().into_bytes())) - .unwrap_or(false) - { - return true.into(); - } - } - false - } - v if !v.is_empty() => ctx - .handle - .block_on(store.key_exists(v.to_string().into_owned().into_bytes())) - .unwrap_or(false), - _ => false, - } - } else { - tracing::warn!( - parent: ctx.span, - context = "sieve:lookup", - event = "failed", - reason = "Unknown lookup id", - lookup_id = ctx.arguments[0].to_string().as_ref(), - ); - false - } - .into() -} - -pub fn exec_get(ctx: PluginContext<'_>) -> Variable { - let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.lookup_stores.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_lookup_store), - }; - - if let Some(store) = store { - ctx.handle - .block_on( - store.key_get::( - ctx.arguments[1].to_string().into_owned().into_bytes(), - ), - ) - .unwrap_or_default() - .map(|v| v.into_inner()) - .unwrap_or_default() - } else { - tracing::warn!( - parent: ctx.span, - context = "sieve:key_get", - event = "failed", - reason = "Unknown store or lookup id", - lookup_id = ctx.arguments[0].to_string().as_ref(), - ); - Variable::default() - } -} - -pub fn exec_set(ctx: PluginContext<'_>) -> Variable { - let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.lookup_stores.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_lookup_store), - }; - - if let Some(store) = store { - let expires = match &ctx.arguments[3] { - Variable::Integer(v) => Some(*v as u64), - Variable::Float(v) => Some(*v as u64), - _ => None, - }; - - ctx.handle - .block_on(store.key_set( - ctx.arguments[1].to_string().into_owned().into_bytes(), - if !ctx.arguments[2].is_empty() { - bincode::serialize(&ctx.arguments[2]).unwrap_or_default() - } else { - vec![] - }, - expires, - )) - .is_ok() - .into() - } else { - tracing::warn!( - parent: ctx.span, - context = "sieve:key_set", - event = "failed", - reason = "Unknown store id", - store_id = ctx.arguments[0].to_string().as_ref(), - ); - Variable::default() - } -} - -pub fn exec_remote(ctx: PluginContext<'_>) -> Variable { - let resource = ctx.arguments[0].to_string(); - let item = ctx.arguments[1].to_string(); - - #[cfg(feature = "test_mode")] - { - if (resource.contains("open") && item.contains("open")) - || (resource.contains("tank") && item.contains("tank")) - { - return true.into(); - } - } - - if resource.is_empty() || item.is_empty() { - return false.into(); - } - - const TIMEOUT: Duration = Duration::from_secs(45); - const RETRY: Duration = Duration::from_secs(3600); - const MAX_ENTRY_SIZE: usize = 256; - const MAX_ENTRIES: usize = 100000; - - match ctx - .core - .sieve - .runtime - .context() - .remote_lists - .lists - .read() - .get(resource.as_ref()) - { - Some(remote_list) if remote_list.expires < Instant::now() => { - return remote_list.entries.contains(item.as_ref()).into() - } - _ => {} - } - - enum Format { - List, - Csv { - column: u32, - separator: char, - skip_first: bool, - }, - } - - // Obtain parameters - let mut format = Format::List; - let mut expires = Duration::from_secs(12 * 3600); - - if let Some(arr) = ctx.arguments[2].as_array() { - // Obtain expiration - match arr.first() { - Some(Variable::Integer(v)) if *v > 0 => { - expires = Duration::from_secs(*v as u64); - } - Some(Variable::Float(v)) if *v > 0.0 => { - expires = Duration::from_secs(*v as u64); - } - _ => (), - } - - // Obtain list type - if matches!(arr.get(1), Some(Variable::String(list_type)) if list_type.eq_ignore_ascii_case("csv")) - { - format = Format::Csv { - column: arr.get(2).map(|v| v.to_integer()).unwrap_or_default() as u32, - separator: arr - .get(3) - .and_then(|v| v.to_string().chars().next()) - .unwrap_or(','), - skip_first: arr.get(4).map_or(false, |v| v.to_bool()), - }; - } - } - - // Lock remote list for writing - let mut _lock = ctx.core.sieve.runtime.context().remote_lists.lists.write(); - let list = _lock - .entry(resource.to_string()) - .or_insert_with(|| RemoteList { - entries: HashSet::new(), - expires: Instant::now(), - }); - - // Make sure that the list is still expired - if list.expires > Instant::now() { - return list.entries.contains(item.as_ref()).into(); - } - - let _enter = ctx.handle.enter(); - match ctx - .handle - .block_on( - reqwest::Client::builder() - .timeout(TIMEOUT) - .user_agent(USER_AGENT) - .build() - .unwrap_or_default() - .get(resource.as_ref()) - .send(), - ) - .and_then(|r| { - if r.status().is_success() { - ctx.handle.block_on(r.bytes()).map(Ok) - } else { - Ok(Err(r)) - } - }) { - Ok(Ok(bytes)) => { - let reader: Box = if resource.ends_with(".gz") { - Box::new(flate2::read::GzDecoder::new(&bytes[..])) - } else { - Box::new(&bytes[..]) - }; - - for (pos, line) in BufReader::new(reader).lines().enumerate() { - match line { - Ok(line_) => { - // Clear list once the first entry has been successfully fetched, decompressed and UTF8-decoded - if pos == 0 { - list.entries.clear(); - } - - match &format { - Format::List => { - let line = line_.trim(); - if !line.is_empty() { - list.entries.insert(line.to_string()); - } - } - Format::Csv { - column, - separator, - skip_first, - } if pos > 0 || !*skip_first => { - let mut in_quote = false; - let mut col_num = 0; - let mut entry = String::new(); - - for ch in line_.chars() { - if ch != '"' { - if ch == *separator && !in_quote { - if col_num == *column { - break; - } else { - col_num += 1; - } - } else if col_num == *column { - entry.push(ch); - if entry.len() > MAX_ENTRY_SIZE { - break; - } - } - } else { - in_quote = !in_quote; - } - } - - if !entry.is_empty() { - list.entries.insert(entry); - } - } - _ => (), - } - } - Err(err) => { - tracing::warn!( - parent: ctx.span, - context = "sieve:key_exists_http", - event = "failed", - resource = resource.as_ref(), - reason = %err, - ); - break; - } - } - - if list.entries.len() == MAX_ENTRIES { - break; - } - } - - tracing::debug!( - parent: ctx.span, - context = "sieve:key_exists_http", - event = "fetch", - resource = resource.as_ref(), - num_entries = list.entries.len(), - ); - - // Update expiration - list.expires = Instant::now() + expires; - return list.entries.contains(item.as_ref()).into(); - } - Ok(Err(response)) => { - tracing::warn!( - parent: ctx.span, - context = "sieve:key_exists_http", - event = "failed", - resource = resource.as_ref(), - status = %response.status(), - ); - } - Err(err) => { - tracing::warn!( - parent: ctx.span, - context = "sieve:key_exists_http", - event = "failed", - resource = resource.as_ref(), - reason = %err, - ); - } - } - - // Something went wrong, try again in one hour - list.expires = Instant::now() + RETRY; - false.into() -} - -pub fn exec_local_domain(ctx: PluginContext<'_>) -> Variable { - let domain = ctx.arguments[0].to_string(); - - if !domain.is_empty() { - let directory = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.directories.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_directory), - }; - - if let Some(directory) = directory { - return ctx - .handle - .block_on(directory.is_local_domain(domain.as_ref())) - .unwrap_or_default() - .into(); - } else { - tracing::warn!( - parent: ctx.span, - context = "sieve:is_local_domain", - event = "failed", - reason = "Unknown directory", - lookup_id = ctx.arguments[0].to_string().as_ref(), - ); - } - } - - Variable::default() -} - -#[derive(Debug, PartialEq, Eq)] -pub struct VariableWrapper(Variable); - -impl Deserialize for VariableWrapper { - fn deserialize(bytes: &[u8]) -> store::Result { - Ok(VariableWrapper( - bincode::deserialize::(bytes).unwrap_or_else(|_| { - Variable::String(String::from_utf8_lossy(bytes).into_owned().into()) - }), - )) - } -} - -impl From for VariableWrapper { - fn from(value: i64) -> Self { - VariableWrapper(value.into()) - } -} - -impl VariableWrapper { - pub fn into_inner(self) -> Variable { - self.0 - } -} - -impl From> for VariableWrapper { - fn from(value: Value<'static>) -> Self { - VariableWrapper(into_sieve_value(value)) - } -} diff --git a/crates/smtp/src/scripts/plugins/mod.rs b/crates/smtp/src/scripts/plugins/mod.rs deleted file mode 100644 index dfb2babb..00000000 --- a/crates/smtp/src/scripts/plugins/mod.rs +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -pub mod bayes; -pub mod dns; -pub mod exec; -pub mod headers; -pub mod http; -pub mod lookup; -pub mod pyzor; -pub mod query; - -use mail_parser::Message; -use sieve::{runtime::Variable, FunctionMap, Input}; -use tokio::runtime::Handle; - -use crate::{config::scripts::SieveContext, core::SMTP}; - -use super::ScriptModification; - -type RegisterPluginFnc = fn(u32, &mut FunctionMap) -> (); -type ExecPluginFnc = fn(PluginContext<'_>) -> Variable; - -pub struct PluginContext<'x> { - pub span: &'x tracing::Span, - pub handle: &'x Handle, - pub core: &'x SMTP, - pub message: &'x Message<'x>, - pub modifications: &'x mut Vec, - pub arguments: Vec, -} - -const PLUGINS_EXEC: [ExecPluginFnc; 16] = [ - query::exec, - exec::exec, - lookup::exec, - lookup::exec_get, - lookup::exec_set, - lookup::exec_remote, - lookup::exec_local_domain, - dns::exec, - dns::exec_exists, - http::exec_header, - bayes::exec_train, - bayes::exec_untrain, - bayes::exec_classify, - bayes::exec_is_balanced, - pyzor::exec, - headers::exec, -]; -const PLUGINS_REGISTER: [RegisterPluginFnc; 16] = [ - query::register, - exec::register, - lookup::register, - lookup::register_get, - lookup::register_set, - lookup::register_remote, - lookup::register_local_domain, - dns::register, - dns::register_exists, - http::register_header, - bayes::register_train, - bayes::register_untrain, - bayes::register_classify, - bayes::register_is_balanced, - pyzor::register, - headers::register, -]; - -pub trait RegisterSievePlugins { - fn register_plugins(self) -> Self; -} - -impl RegisterSievePlugins for FunctionMap { - fn register_plugins(mut self) -> Self { - #[cfg(feature = "test_mode")] - { - self.set_external_function("print", PLUGINS_EXEC.len() as u32, 1) - } - - for (i, fnc) in PLUGINS_REGISTER.iter().enumerate() { - fnc(i as u32, &mut self); - } - self - } -} - -impl SMTP { - pub fn run_plugin_blocking(&self, id: u32, ctx: PluginContext<'_>) -> Input { - #[cfg(feature = "test_mode")] - if id == PLUGINS_EXEC.len() as u32 { - return test_print(ctx); - } - - PLUGINS_EXEC - .get(id as usize) - .map(|fnc| fnc(ctx)) - .unwrap_or_default() - .into() - } -} - -#[cfg(feature = "test_mode")] -pub fn test_print(ctx: PluginContext<'_>) -> Input { - println!("{}", ctx.arguments[0].to_string()); - Input::True -} diff --git a/crates/smtp/src/scripts/plugins/pyzor.rs b/crates/smtp/src/scripts/plugins/pyzor.rs deleted file mode 100644 index e95ed09f..00000000 --- a/crates/smtp/src/scripts/plugins/pyzor.rs +++ /dev/null @@ -1,836 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use sieve::{runtime::Variable, FunctionMap}; - -use crate::config::scripts::SieveContext; - -use super::PluginContext; - -use std::{ - borrow::Cow, - io::Write, - time::{Duration, SystemTime}, -}; - -use mail_parser::{decoders::html::add_html_token, Message, PartType}; -use nlp::tokenizers::types::{TokenType, TypesTokenizer}; -use sha1::{Digest, Sha1}; -use tokio::net::UdpSocket; -use utils::suffixlist::PublicSuffix; - -const MIN_LINE_LENGTH: usize = 8; -const ATOMIC_NUM_LINES: usize = 4; -const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)]; - -#[derive(Default, Debug, PartialEq, Eq)] -struct PyzorResponse { - code: u32, - count: u64, - wl_count: u64, -} - -pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("pyzor_check", plugin_id, 2); -} - -pub fn exec(ctx: PluginContext<'_>) -> Variable { - // Make sure there is at least one text part - if !ctx - .message - .parts - .iter() - .any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_))) - { - return Variable::default(); - } - - // Hash message - let request = ctx - .message - .pyzor_check_message(&ctx.core.sieve.runtime.context().psl); - - #[cfg(feature = "test_mode")] - { - if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") { - return PyzorResponse { - code: 200, - count: 1000, - wl_count: 0, - } - .into(); - } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") { - return PyzorResponse { - code: 200, - count: 60, - wl_count: 10, - } - .into(); - } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") { - return PyzorResponse { - code: 200, - count: 50, - wl_count: 20, - } - .into(); - } - } - - let span = ctx.span; - let address = ctx.arguments[0].to_string(); - let timeout = Duration::from_secs(std::cmp::max( - std::cmp::min(ctx.arguments[1].to_integer() as u64, 60), - 5, - )); - // Send message to address - match ctx - .handle - .block_on(pyzor_send_message(address.as_ref(), timeout, &request)) - { - Ok(response) => response.into(), - Err(err) => { - tracing::debug!( - parent: span, - context = "sieve:pyzor_check", - event = "failed", - reason = %err, - ); - Variable::default() - } - } -} - -impl From for Variable { - fn from(response: PyzorResponse) -> Self { - vec![ - Variable::from(response.code), - Variable::from(response.count), - Variable::from(response.wl_count), - ] - .into() - } -} - -async fn pyzor_send_message( - addr: &str, - timeout: Duration, - message: &str, -) -> std::io::Result { - let socket = UdpSocket::bind("0.0.0.0:0").await?; - tokio::time::timeout(timeout, socket.send_to(message.as_bytes(), addr)).await??; - - let mut buffer = vec![0u8; 1024]; - let (size, _) = tokio::time::timeout(timeout, socket.recv_from(&mut buffer)).await??; - - let raw_response = std::str::from_utf8(&buffer[..size]) - .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; - let mut response = PyzorResponse { - code: u32::MAX, - count: u64::MAX, - wl_count: u64::MAX, - }; - - for line in raw_response.lines() { - if let Some((k, v)) = line.split_once(':') { - if k.eq_ignore_ascii_case("code") { - response.code = v.trim().parse().map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid line: {raw_response}"), - ) - })?; - } else if k.eq_ignore_ascii_case("count") { - response.count = v.trim().parse().map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid line: {raw_response}"), - ) - })?; - } else if k.eq_ignore_ascii_case("wl-count") { - response.wl_count = v.trim().parse().map_err(|_| { - std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid line: {raw_response}"), - ) - })?; - } - } - } - - if response.code != u32::MAX && response.count != u64::MAX && response.wl_count != u64::MAX { - Ok(response) - } else { - Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - format!("Invalid response: {raw_response}"), - )) - } -} - -trait PyzorDigest { - fn pyzor_digest(&self, writer: W, psl: &PublicSuffix) -> W; -} - -pub trait PyzorCheck { - fn pyzor_check_message(&self, psl: &PublicSuffix) -> String; -} - -impl<'x, W: Write> PyzorDigest for Message<'x> { - fn pyzor_digest(&self, writer: W, psl: &PublicSuffix) -> W { - let parts = self - .parts - .iter() - .filter_map(|part| match &part.body { - PartType::Text(text) => Some(text.as_ref().into()), - PartType::Html(html) => Some(html_to_text(html.as_ref()).into()), - _ => None, - }) - .collect::>>(); - - pyzor_digest(writer, parts.iter().flat_map(|text| text.lines()), psl) - } -} - -impl<'x> PyzorCheck for Message<'x> { - fn pyzor_check_message(&self, psl: &PublicSuffix) -> String { - let time = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); - - pyzor_create_message( - self, - psl, - time, - (time & 0xFFFF) as u16 ^ ((time >> 16) & 0xFFFF) as u16, - ) - } -} - -fn pyzor_create_message( - message: &Message<'_>, - psl: &PublicSuffix, - time: u64, - thread: u16, -) -> String { - // Hash message - let hash = message.pyzor_digest(Sha1::new(), psl).finalize(); - // Hash key - let mut hash_key = Sha1::new(); - hash_key.update("anonymous:".as_bytes()); - let hash_key = hash_key.finalize(); - - // Hash message - let message = format!( - "Op: check\nOp-Digest: {hash:x}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}" - ); - let mut msg_hash = Sha1::new(); - msg_hash.update(message.as_bytes()); - let msg_hash = msg_hash.finalize(); - - // Sign - let mut sig = Sha1::new(); - sig.update(msg_hash); - sig.update(&format!(":{time}:{hash_key:x}")); - let sig = sig.finalize(); - - format!("{message}\nSig: {sig:x}\n") -} - -fn pyzor_digest<'x, I, W>(mut writer: W, lines: I, psl: &PublicSuffix) -> W -where - I: Iterator, - W: Write, -{ - let mut result = Vec::with_capacity(16); - - for line in lines { - let mut clean_line = String::with_capacity(line.len()); - let mut token_start = usize::MAX; - let mut token_end = usize::MAX; - - let add_line = |line: &mut String, span: &str| { - if !span.contains(char::from(0)) { - if span.len() < 10 { - line.push_str(span); - } - } else { - let span = span.replace(char::from(0), ""); - if span.len() < 10 { - line.push_str(&span); - } - } - }; - - for token in TypesTokenizer::new(line, psl) { - match token.word { - TokenType::Alphabetic(_) - | TokenType::Alphanumeric(_) - | TokenType::Integer(_) - | TokenType::Float(_) - | TokenType::Other(_) - | TokenType::Punctuation(_) => { - if token_start == usize::MAX { - token_start = token.from; - } - token_end = token.to; - } - TokenType::Space - | TokenType::Url(_) - | TokenType::UrlNoScheme(_) - | TokenType::UrlNoHost(_) - | TokenType::Email(_) => { - if token_start != usize::MAX { - add_line(&mut clean_line, &line[token_start..token_end]); - token_start = usize::MAX; - token_end = usize::MAX; - } - } - } - } - - if token_start != usize::MAX { - add_line(&mut clean_line, &line[token_start..token_end]); - } - - if clean_line.len() >= MIN_LINE_LENGTH { - result.push(clean_line); - } - } - - if result.len() > ATOMIC_NUM_LINES { - for (offset, length) in DIGEST_SPEC { - for i in 0..*length { - if let Some(line) = result.get((*offset * result.len() / 100) + i) { - let _ = writer.write_all(line.as_bytes()); - } - } - } - } else { - for line in result { - let _ = writer.write_all(line.as_bytes()); - } - } - - writer -} - -fn html_to_text(input: &str) -> String { - let mut result = String::with_capacity(input.len()); - let input = input.as_bytes(); - - let mut in_tag = false; - let mut in_comment = false; - let mut in_style = false; - let mut in_script = false; - - let mut is_token_start = true; - let mut is_after_space = false; - let mut is_tag_close = false; - - let mut token_start = 0; - let mut token_end = 0; - - let mut tag_token_pos = 0; - let mut comment_pos = 0; - - for (pos, ch) in input.iter().enumerate() { - if !in_comment { - match ch { - b'<' => { - if !(in_tag || in_style || in_script || is_token_start) { - add_html_token( - &mut result, - &input[token_start..token_end + 1], - is_after_space, - ); - is_after_space = false; - } - - tag_token_pos = 0; - in_tag = true; - is_token_start = true; - is_tag_close = false; - continue; - } - b'>' if in_tag => { - if tag_token_pos == 1 { - if let Some(tag) = input.get(token_start..token_end + 1) { - if tag.eq_ignore_ascii_case(b"style") { - in_style = !is_tag_close; - } else if tag.eq_ignore_ascii_case(b"script") { - in_script = !is_tag_close; - } - } - } - - in_tag = false; - is_token_start = true; - is_after_space = !result.is_empty(); - - continue; - } - b'/' if in_tag => { - if tag_token_pos == 0 { - is_tag_close = true; - } - continue; - } - b'!' if in_tag && tag_token_pos == 0 => { - if let Some(b"--") = input.get(pos + 1..pos + 3) { - in_comment = true; - continue; - } - } - b' ' | b'\t' | b'\r' | b'\n' => { - if !(in_tag || in_style || in_script) { - if !is_token_start { - add_html_token( - &mut result, - &input[token_start..token_end + 1], - is_after_space, - ); - } - is_after_space = true; - } - - is_token_start = true; - continue; - } - b'&' if !(in_tag || is_token_start || in_style || in_script) => { - add_html_token( - &mut result, - &input[token_start..token_end + 1], - is_after_space, - ); - is_token_start = true; - is_after_space = false; - } - b';' if !(in_tag || is_token_start || in_style || in_script) => { - add_html_token(&mut result, &input[token_start..pos + 1], is_after_space); - is_token_start = true; - is_after_space = false; - continue; - } - _ => (), - } - if is_token_start { - token_start = pos; - is_token_start = false; - if in_tag { - tag_token_pos += 1; - } - } - token_end = pos; - } else { - match ch { - b'-' => comment_pos += 1, - b'>' if comment_pos == 2 => { - comment_pos = 0; - in_comment = false; - in_tag = false; - is_token_start = true; - } - _ => comment_pos = 0, - } - } - } - - if !(in_tag || is_token_start || in_style || in_script) { - add_html_token( - &mut result, - &input[token_start..token_end + 1], - is_after_space, - ); - } - - result.shrink_to_fit(); - result -} - -#[cfg(test)] -mod test { - use std::time::Duration; - - use mail_parser::MessageParser; - use sha1::Digest; - use sha1::Sha1; - use utils::suffixlist::PublicSuffix; - - use super::pyzor_create_message; - use super::pyzor_send_message; - use super::{html_to_text, pyzor_digest, PyzorDigest}; - - use super::PyzorResponse; - - #[ignore] - #[tokio::test] - async fn send_message() { - assert_eq!( - pyzor_send_message( - "public.pyzor.org:24441", - Duration::from_secs(10), - concat!( - "Op: check\n", - "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", - "Thread: 49005\n", - "PV: 2.1\n", - "User: anonymous\n", - "Time: 1697468672\n", - "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" - ), - ) - .await - .unwrap(), - PyzorResponse { - code: 200, - count: 0, - wl_count: 0 - } - ); - } - - #[test] - fn message_pyzor() { - let mut psl = PublicSuffix::default(); - psl.suffixes.insert("com".to_string()); - let message = pyzor_create_message( - &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(), - &psl, - 1697468672, - 49005, - ); - - assert_eq!( - message, - concat!( - "Op: check\n", - "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", - "Thread: 49005\n", - "PV: 2.1\n", - "User: anonymous\n", - "Time: 1697468672\n", - "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" - ) - ); - } - - #[test] - fn digest_pyzor() { - let mut psl = PublicSuffix::default(); - psl.suffixes.insert("com".to_string()); - - // HTML stripping - assert_eq!(html_to_text(HTML_RAW), HTML_RAW_STRIPED); - - // Token stripping - for strip_me in [ - "t@abc.com", - "t1@abc.com", - "t+a@abc.com", - "t.a@abc.com", - "0A2D3f%a#S", - "3sddkf9jdkd9", - "@@#@@@@@@@@@", - "http://spammer.com/special-offers?buy=now", - ] { - assert_eq!( - String::from_utf8(pyzor_digest( - Vec::new(), - format!("Test {strip_me} Test2").lines(), - &psl - )) - .unwrap(), - "TestTest2" - ); - } - - // Test short lines - assert_eq!( - String::from_utf8(pyzor_digest( - Vec::new(), - concat!("This line is included\n", "not this\n", "This also").lines(), - &psl - )) - .unwrap(), - "ThislineisincludedThisalso" - ); - - // Test atomic - assert_eq!( - String::from_utf8(pyzor_digest( - Vec::new(), - "All this message\nShould be included\nIn the digest".lines(), - &psl - )) - .unwrap(), - "AllthismessageShouldbeincludedInthedigest" - ); - - // Test spec - let mut text = String::new(); - for i in 0..100 { - text += &format!("Line{i} test test test\n"); - } - let mut expected = String::new(); - for i in [20, 21, 22, 60, 61, 62] { - expected += &format!("Line{i}testtesttest"); - } - assert_eq!( - String::from_utf8(pyzor_digest(Vec::new(), text.lines(), &psl)).unwrap(), - expected - ); - - // Test email parsing - for (input, expected) in [ - ( - HTML_TEXT, - concat!( - "Emailspam,alsoknownasjunkemailorbulkemail,isasubset", - "ofspaminvolvingnearlyidenticalmessagessenttonumerous", - "byemail.Clickingonlinksinspamemailmaysendusersto", - "byemail.Clickingonlinksinspamemailmaysendusersto", - "phishingwebsitesorsitesthatarehostingmalware.", - "Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,", - "isasubsetofspaminvolvingnearlyidenticalmessage", - "ssenttonumerousbyemail.Clickingonlinksinspamemailmaysenduse", - "rstophishingwebsitesorsitesthatarehostingmalware." - ), - ), - (HTML_TEXT_STYLE_SCRIPT, "Thisisatest.Thisisatest."), - (TEXT_ATTACHMENT, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_NULL, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_MULTIPLE_NULLS, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_SUBJECT_NULL, "Thisisatestmailing"), - (TEXT_ATTACHMENT_W_CONTENTTYPE_NULL, "Thisisatestmailing"), - ] { - assert_eq!( - String::from_utf8( - MessageParser::new() - .parse(input) - .unwrap() - .pyzor_digest(Vec::new(), &psl) - ) - .unwrap(), - expected, - "failed for {input}" - ) - } - - // Test SHA hash - assert_eq!( - format!( - "{:x}", - MessageParser::new() - .parse(HTML_TEXT_STYLE_SCRIPT) - .unwrap() - .pyzor_digest(Sha1::new(), &psl) - .finalize() - ), - "b2c27325a034c581df0c9ef37e4a0d63208a3e7e", - ) - } - - const HTML_TEXT: &str = r#"MIME-Version: 1.0 -Sender: chirila@gapps.spamexperts.com -Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) -Date: Thu, 16 Jan 2014 10:43:31 +0200 -Delivered-To: chirila@gapps.spamexperts.com -X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd - ---001a11c25ff293069304f0126bfd -Content-Type: text/plain; charset=ISO-8859-1 - -Email spam. - -Email spam, also known as junk email or unsolicited bulk email, is a subset -of electronic spam involving nearly identical messages sent to numerous -recipients by email. Clicking on links in spam email may send users to -phishing web sites or sites that are hosting malware. - ---001a11c25ff293069304f0126bfd -Content-Type: text/html; charset=ISO-8859-1 -Content-Transfer-Encoding: quoted-printable - -
Email spam.

Email spam, also= - known as junk email or unsolicited bulk email, is a subset of electronic s= -pam involving nearly identical messages sent to numerous recipients by emai= -l. Clicking on links in spam email may send users to phishing web sites or = -sites that are hosting malware.
-
- ---001a11c25ff293069304f0126bfd-- -"#; - - const HTML_TEXT_STYLE_SCRIPT: &str = r#"MIME-Version: 1.0 -Sender: chirila@gapps.spamexperts.com -Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) -Date: Thu, 16 Jan 2014 10:43:31 +0200 -Delivered-To: chirila@gapps.spamexperts.com -X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd - ---001a11c25ff293069304f0126bfd -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test. - ---001a11c25ff293069304f0126bfd -Content-Type: text/html; charset=ISO-8859-1 -Content-Transfer-Encoding: quoted-printable - -
- - -
This is a test.
-
- ---001a11c25ff293069304f0126bfd-- -"#; - - const TEXT_ATTACHMENT: &str = r#"MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test mailing - ---f46d040a62c49bb1c404f027e8ca-- ---f46d040a62c49bb1c804f027e8cc -Content-Type: image/png; name="tar.png" -Content-Disposition: attachment; filename="tar.png" -Content-Transfer-Encoding: base64 -X-Attachment-Id: f_hqjas5ad0 - -iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD -QmCC ---f46d040a62c49bb1c804f027e8cc--"#; - - const TEXT_ATTACHMENT_W_NULL: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test ma\0iling ---f46d040a62c49bb1c804f027e8cc--"; - - const TEXT_ATTACHMENT_W_MULTIPLE_NULLS: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test ma\0\0\0iling ---f46d040a62c49bb1c804f027e8cc--"; - - const TEXT_ATTACHMENT_W_SUBJECT_NULL: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Te\0\0\0st -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=ISO-8859-1 - -This is a test mailing ---f46d040a62c49bb1c804f027e8cc--"; - - const TEXT_ATTACHMENT_W_CONTENTTYPE_NULL: &str = "MIME-Version: 1.0 -Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) -Date: Fri, 17 Jan 2014 12:21:43 +0200 -Delivered-To: chirila.s.alexandru@gmail.com -Message-ID: -Subject: Test -From: Alexandru Chirila -To: Alexandru Chirila -Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc - ---f46d040a62c49bb1c804f027e8cc -Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca - ---f46d040a62c49bb1c404f027e8ca -Content-Type: text/plain; charset=\"iso-8859-1\0\0\0\" - -This is a test mailing ---f46d040a62c49bb1c804f027e8cc--"; - - const HTML_RAW: &str = r#"Email spam -

Email spam, also known as junk email -or unsolicited bulk email (UBE), is a subset of -electronic spam -involving nearly identical messages sent to numerous recipients by -email. Clicking on -links in spam email may send users to phishing -web sites or sites that are hosting malware."#; - - const HTML_RAW_STRIPED : &str = concat!("Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE )," , - " is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email" , - " . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware ."); -} diff --git a/crates/smtp/src/scripts/plugins/query.rs b/crates/smtp/src/scripts/plugins/query.rs deleted file mode 100644 index da7bcde8..00000000 --- a/crates/smtp/src/scripts/plugins/query.rs +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level store of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::cmp::Ordering; - -use crate::{ - config::scripts::SieveContext, - core::{into_sieve_value, to_store_value}, -}; -use sieve::{runtime::Variable, FunctionMap}; -use store::{Rows, Value}; - -use super::PluginContext; - -pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { - fnc_map.set_external_function("query", plugin_id, 3); -} - -pub fn exec(ctx: PluginContext<'_>) -> Variable { - let span = ctx.span; - - // Obtain store name - let store = match &ctx.arguments[0] { - Variable::String(v) if !v.is_empty() => ctx.core.shared.lookup_stores.get(v.as_ref()), - _ => Some(&ctx.core.shared.default_lookup_store), - }; - - let store = if let Some(store) = store { - store - } else { - tracing::warn!( - parent: span, - context = "sieve:query", - event = "failed", - reason = "Unknown store", - store = ctx.arguments[0].to_string().as_ref(), - ); - return false.into(); - }; - - // Obtain query string - let query = ctx.arguments[1].to_string(); - if query.is_empty() { - tracing::warn!( - parent: span, - context = "sieve:query", - event = "invalid", - reason = "Empty query string", - ); - return false.into(); - } - - // Obtain arguments - let arguments = match &ctx.arguments[2] { - Variable::Array(l) => l.iter().map(to_store_value).collect(), - v => vec![to_store_value(v)], - }; - - // Run query - if query - .as_bytes() - .get(..6) - .map_or(false, |q| q.eq_ignore_ascii_case(b"SELECT")) - { - if let Ok(mut rows) = ctx.handle.block_on(store.query::(&query, arguments)) { - match rows.rows.len().cmp(&1) { - Ordering::Equal => { - let mut row = rows.rows.pop().unwrap().values; - match row.len().cmp(&1) { - Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { - row.pop().map(into_sieve_value).unwrap() - } - Ordering::Less => Variable::default(), - _ => Variable::Array( - row.into_iter() - .map(into_sieve_value) - .collect::>() - .into(), - ), - } - } - Ordering::Less => Variable::default(), - Ordering::Greater => rows - .rows - .into_iter() - .map(|r| { - Variable::Array( - r.values - .into_iter() - .map(into_sieve_value) - .collect::>() - .into(), - ) - }) - .collect::>() - .into(), - } - } else { - false.into() - } - } else { - ctx.handle - .block_on(store.query::(&query, arguments)) - .is_ok() - .into() - } -} diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 9ceb2ffd..cee7a73e 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -289,229 +289,6 @@ impl Stores { } } -#[allow(async_fn_in_trait)] -pub trait ConfigStore { - async fn parse_stores(&mut self) -> utils::config::Result; - async fn parse_purge_schedules( - &self, - stores: &Stores, - store: Option<&str>, - blob_store: Option<&str>, - ) -> utils::config::Result>; -} - -impl ConfigStore for Config { - #[allow(unused_variables)] - #[allow(unreachable_code)] - async fn parse_stores(&mut self) -> utils::config::Result { - let mut config = Stores::default(); - - let ids = self - .sub_keys("store", ".type") - .map(|id| id.to_string()) - .collect::>(); - for id in ids { - let id = id.as_str(); - // Parse store - if self.property_or_default::(("store", id, "disable"), "false")? { - tracing::debug!("Skipping disabled store {id:?}."); - continue; - } - let protocol = self - .value_require(("store", id, "type"))? - .to_ascii_lowercase(); - let prefix = ("store", id); - let store_id = id.to_string(); - let compression_algo = - self.property_or_default::(("store", id, "compression"), "lz4")?; - - let lookup_store: Store = match protocol.as_str() { - #[cfg(feature = "rocks")] - "rocksdb" => { - let db: Store = RocksDbStore::open(self, prefix).await.unwrap().into(); - config.stores.insert(store_id.clone(), db.clone()); - config - .fts_stores - .insert(store_id.clone(), db.clone().into()); - config.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - config.lookup_stores.insert(store_id, db.into()); - continue; - } - #[cfg(feature = "foundation")] - "foundationdb" => { - let db: Store = FdbStore::open(self, prefix).await.unwrap().into(); - config.stores.insert(store_id.clone(), db.clone()); - config - .fts_stores - .insert(store_id.clone(), db.clone().into()); - config.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - config.lookup_stores.insert(store_id, db.into()); - continue; - } - #[cfg(feature = "postgres")] - "postgresql" => { - let db: Store = PostgresStore::open(self, prefix).await.unwrap().into(); - config.stores.insert(store_id.clone(), db.clone()); - config - .fts_stores - .insert(store_id.clone(), db.clone().into()); - config.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - db - } - #[cfg(feature = "mysql")] - "mysql" => { - let db: Store = MysqlStore::open(self, prefix).await.unwrap().into(); - config.stores.insert(store_id.clone(), db.clone()); - config - .fts_stores - .insert(store_id.clone(), db.clone().into()); - config.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - db - } - #[cfg(feature = "sqlite")] - "sqlite" => { - let db: Store = SqliteStore::open(self, prefix).unwrap().into(); - config.stores.insert(store_id.clone(), db.clone()); - config - .fts_stores - .insert(store_id.clone(), db.clone().into()); - config.blob_stores.insert( - store_id.clone(), - BlobStore::from(db.clone()).with_compression(compression_algo), - ); - db - } - "fs" => { - config.blob_stores.insert( - store_id, - BlobStore::from(FsStore::open(self, prefix).await.unwrap()) - .with_compression(compression_algo), - ); - continue; - } - #[cfg(feature = "s3")] - "s3" => { - config.blob_stores.insert( - store_id, - BlobStore::from(S3Store::open(self, prefix).await.unwrap()) - .with_compression(compression_algo), - ); - continue; - } - #[cfg(feature = "elastic")] - "elasticsearch" => { - config.fts_stores.insert( - store_id, - ElasticSearchStore::open(self, prefix).await.unwrap().into(), - ); - continue; - } - #[cfg(feature = "redis")] - "redis" => { - config.lookup_stores.insert( - store_id, - RedisStore::open(self, prefix).await.unwrap().into(), - ); - continue; - } - unknown => { - tracing::debug!("Unknown directory type: {unknown:?}"); - continue; - } - }; - - // Add queries as lookup stores - let lookup_store: LookupStore = lookup_store.into(); - for lookup_id in self.sub_keys(("store", id, "query"), "") { - config.lookup_stores.insert( - format!("{store_id}/{lookup_id}"), - LookupStore::Query(Arc::new(QueryStore { - store: lookup_store.clone(), - query: self.property_require(("store", id, "query", lookup_id))?, - })), - ); - } - config.lookup_stores.insert(store_id, lookup_store.clone()); - - // Run init queries on database - for (_, query) in self.values(("store", id, "init.execute")) { - if let Err(err) = lookup_store.query::(query, Vec::new()).await { - tracing::warn!("Failed to initialize store {id:?}: {err}"); - } - } - } - - Ok(config) - } - - async fn parse_purge_schedules( - &self, - stores: &Stores, - store_id: Option<&str>, - blob_store_id: Option<&str>, - ) -> utils::config::Result> { - let mut schedules = Vec::new(); - let remove = "true"; - - if let Some(store) = store_id.and_then(|store_id| stores.stores.get(store_id)) { - let store_id = store_id.unwrap(); - if let Some(cron) = - self.property::(("store", store_id, "purge.frequency"))? - { - schedules.push(PurgeSchedule { - cron, - store_id: store_id.to_string(), - store: PurgeStore::Data(store.clone()), - }); - } - - if let Some(blob_store) = - blob_store_id.and_then(|blob_store_id| stores.blob_stores.get(blob_store_id)) - { - let blob_store_id = blob_store_id.unwrap(); - if let Some(cron) = - self.property::(("store", blob_store_id, "purge.frequency"))? - { - schedules.push(PurgeSchedule { - cron, - store_id: blob_store_id.to_string(), - store: PurgeStore::Blobs { - store: store.clone(), - blob_store: blob_store.clone(), - }, - }); - } - } - } - - for (store_id, store) in &stores.lookup_stores { - if let Some(cron) = - self.property::(("store", store_id.as_str(), "purge.frequency"))? - { - schedules.push(PurgeSchedule { - cron, - store_id: store_id.clone(), - store: PurgeStore::Lookup(store.clone()), - }); - } - } - - Ok(schedules) - } -} - impl From for String { fn from(err: crate::Error) -> Self { match err { diff --git a/crates/store/src/dispatch/blob.rs b/crates/store/src/dispatch/blob.rs index a80661b8..b17ea754 100644 --- a/crates/store/src/dispatch/blob.rs +++ b/crates/store/src/dispatch/blob.rs @@ -50,6 +50,7 @@ impl BlobStore { Store::MySQL(store) => store.get_blob(key, read_range).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.get_blob(key, read_range).await, + Store::None => Err(crate::Error::InternalError("No store configured".into())), }, BlobBackend::Fs(store) => store.get_blob(key, read_range).await, #[cfg(feature = "s3")] @@ -115,6 +116,7 @@ impl BlobStore { Store::MySQL(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.put_blob(key, data.as_ref()).await, + Store::None => Err(crate::Error::InternalError("No store configured".into())), }, BlobBackend::Fs(store) => store.put_blob(key, data.as_ref()).await, #[cfg(feature = "s3")] @@ -135,6 +137,7 @@ impl BlobStore { Store::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Store::RocksDb(store) => store.delete_blob(key).await, + Store::None => Err(crate::Error::InternalError("No store configured".into())), }, BlobBackend::Fs(store) => store.delete_blob(key).await, #[cfg(feature = "s3")] diff --git a/crates/store/src/dispatch/blocked.rs b/crates/store/src/dispatch/blocked.rs deleted file mode 100644 index 1b8bfea3..00000000 --- a/crates/store/src/dispatch/blocked.rs +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - fmt::Debug, - net::IpAddr, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; - -use ahash::AHashSet; -use arc_swap::{ArcSwap, ArcSwapOption}; -use parking_lot::RwLock; -use utils::config::{ipmask::IpAddrMask, utils::ParseKey, Config, ConfigKey, Rate}; - -use crate::LookupStore; - -pub struct BlockedIps { - ip_addresses: RwLock>, - ip_networks: ArcSwap>, - has_networks: AtomicBool, - store: LookupStore, - limiter_rate: ArcSwapOption, -} - -pub const BLOCKED_IP_KEY: &str = "server.security.blocked-networks"; -pub const BLOCKED_IP_PREFIX: &str = "server.security.blocked-networks."; - -impl BlockedIps { - pub fn new(store: LookupStore) -> Self { - Self { - ip_addresses: RwLock::new(AHashSet::new()), - ip_networks: ArcSwap::new(Arc::new(Vec::new())), - has_networks: AtomicBool::new(false), - limiter_rate: ArcSwapOption::empty(), - store, - } - } - - pub fn reload(&self, config: &Config) -> utils::config::Result<()> { - self.limiter_rate.store( - config - .property::("authentication.fail2ban")? - .map(Arc::new), - ); - self.reload_blocked_ips(config.set_values(BLOCKED_IP_KEY)) - } - - pub fn reload_blocked_ips(&self, ips: I) -> utils::config::Result<()> - where - T: AsRef, - I: IntoIterator, - { - let mut ip_addresses = AHashSet::new(); - let mut ip_networks = Vec::new(); - - for ip in ips { - let ip = ip.as_ref(); - if ip.contains('/') { - ip_networks.push(ip.parse_key(BLOCKED_IP_KEY)?); - } else { - ip_addresses.insert(ip.parse_key(BLOCKED_IP_KEY)?); - } - } - - self.has_networks - .store(!ip_networks.is_empty(), Ordering::Relaxed); - *self.ip_addresses.write() = ip_addresses; - self.ip_networks.store(Arc::new(ip_networks)); - - Ok(()) - } - - pub async fn is_fail2banned(&self, ip: IpAddr, login: String) -> Option { - if let Some(rate) = self.limiter_rate.load().as_ref() { - let is_allowed = self - .store - .is_rate_allowed(format!("b:{}", ip).as_bytes(), rate.as_ref(), false) - .await - .map(|v| v.is_none()) - .unwrap_or(false) - && self - .store - .is_rate_allowed(format!("b:{}", login).as_bytes(), rate.as_ref(), false) - .await - .map(|v| v.is_none()) - .unwrap_or(false); - if !is_allowed { - self.ip_addresses.write().insert(ip); - return Some(ConfigKey { - key: format!("{}.{}", BLOCKED_IP_KEY, ip), - value: String::new(), - }); - } - } - - None - } - - pub fn has_fail2ban(&self) -> bool { - self.limiter_rate.load().is_some() - } - - pub fn is_blocked(&self, ip: &IpAddr) -> bool { - self.ip_addresses.read().contains(ip) - || (self.has_networks.load(Ordering::Relaxed) - && self - .ip_networks - .load() - .iter() - .any(|network| network.matches(ip))) - } -} - -impl Debug for BlockedIps { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("BlockedIps") - .field("ip_addresses", &self.ip_addresses) - .field("ip_networks", &self.ip_networks) - .field("limiter_rate", &self.limiter_rate) - .finish() - } -} diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 1b032982..58a98fb3 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -21,7 +21,7 @@ * for more details. */ -use utils::{config::Rate, expr}; +use utils::config::Rate; use crate::{write::LookupClass, Row}; #[allow(unused_imports)] @@ -383,19 +383,3 @@ impl From> for String { } } } - -impl<'x> From> for expr::Variable<'x> { - fn from(value: Value<'x>) -> Self { - match value { - Value::Integer(v) => expr::Variable::Integer(v), - Value::Bool(v) => expr::Variable::Integer(v as i64), - Value::Float(v) => expr::Variable::Float(v), - Value::Text(v) => expr::Variable::String(v), - Value::Blob(v) => expr::Variable::String(match v { - std::borrow::Cow::Borrowed(v) => String::from_utf8_lossy(v), - std::borrow::Cow::Owned(v) => String::from_utf8_lossy(&v).into_owned().into(), - }), - Value::Null => expr::Variable::String("".into()), - } - } -} diff --git a/crates/store/src/dispatch/mod.rs b/crates/store/src/dispatch/mod.rs index 1a361edb..c4641e59 100644 --- a/crates/store/src/dispatch/mod.rs +++ b/crates/store/src/dispatch/mod.rs @@ -22,7 +22,6 @@ */ pub mod blob; -pub mod blocked; pub mod config; pub mod fts; pub mod lookup; diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index e5f2eec4..0667e890 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -53,6 +53,7 @@ impl Store { Self::MySQL(store) => store.get_value(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_value(key).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -71,6 +72,7 @@ impl Store { Self::MySQL(store) => store.get_bitmap(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_bitmap(key).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -112,6 +114,7 @@ impl Store { Self::MySQL(store) => store.iterate(params, cb).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.iterate(params, cb).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -130,6 +133,7 @@ impl Store { Self::MySQL(store) => store.get_counter(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_counter(key).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -185,6 +189,7 @@ impl Store { Self::MySQL(store) => store.write(batch).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.write(batch).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), }?; for (key, class, document_id, set) in bitmaps { @@ -225,6 +230,7 @@ impl Store { Self::MySQL(store) => store.write(batch).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.write(batch).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -267,6 +273,7 @@ impl Store { Self::MySQL(store) => store.purge_store().await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.purge_store().await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -282,6 +289,7 @@ impl Store { Self::MySQL(store) => store.delete_range(from, to).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.delete_range(from, to).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -342,6 +350,7 @@ impl Store { Self::MySQL(store) => store.get_blob(key, range).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.get_blob(key, range).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -357,6 +366,7 @@ impl Store { Self::MySQL(store) => store.put_blob(key, data).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.put_blob(key, data).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } @@ -372,6 +382,7 @@ impl Store { Self::MySQL(store) => store.delete_blob(key).await, #[cfg(feature = "rocks")] Self::RocksDb(store) => store.delete_blob(key).await, + Self::None => Err(crate::Error::InternalError("No store configured".into())), } } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 4a740f96..e57cc39e 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -193,7 +193,7 @@ pub struct Stores { pub purge_schedules: Vec, } -#[derive(Clone)] +#[derive(Clone, Default)] pub enum Store { #[cfg(feature = "sqlite")] SQLite(Arc), @@ -205,6 +205,8 @@ pub enum Store { MySQL(Arc), #[cfg(feature = "rocks")] RocksDb(Arc), + #[default] + None, } #[derive(Clone)] @@ -336,6 +338,27 @@ impl From for LookupStore { } } +impl Default for BlobStore { + fn default() -> Self { + Self { + backend: BlobBackend::Store(Store::None), + compression: CompressionAlgo::None, + } + } +} + +impl Default for LookupStore { + fn default() -> Self { + Self::Store(Store::None) + } +} + +impl Default for FtsStore { + fn default() -> Self { + Self::Store(Store::None) + } +} + #[derive(Clone, Debug, PartialEq)] pub enum Value<'x> { Integer(i64), @@ -662,20 +685,7 @@ impl std::fmt::Debug for Store { Self::MySQL(_) => f.debug_tuple("MySQL").finish(), #[cfg(feature = "rocks")] Self::RocksDb(_) => f.debug_tuple("RocksDb").finish(), - } - } -} - -#[cfg(feature = "test_mode")] -impl Default for Store { - fn default() -> Self { - #[cfg(feature = "sqlite")] - { - Self::SQLite(Arc::new(SqliteStore::open_memory().unwrap())) - } - #[cfg(not(feature = "sqlite"))] - { - unreachable!("No default store available") + Self::None => f.debug_tuple("None").finish(), } } } diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 66154b40..4e301cea 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -15,13 +15,6 @@ tracing = "0.1" mail-auth = { version = "0.3" } smtp-proto = { version = "0.1" } mail-send = { version = "0.4", default-features = false, features = ["cram-md5"] } -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tracing-appender = "0.2" -tracing-opentelemetry = "0.22.0" -opentelemetry = { version = "0.21.0" } -opentelemetry_sdk = { version = "0.21.0", features = ["rt-tokio"] } -opentelemetry-otlp = { version = "0.14.0", features = ["http-proto", "reqwest-client"] } -opentelemetry-semantic-conventions = { version = "0.13.0" } dashmap = "5.4" ahash = { version = "0.8" } chrono = "0.4" @@ -35,9 +28,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls- x509-parser = "0.16.0" pem = "3.0" parking_lot = "0.12" -arc-swap = "1.6.0" futures = "0.3" -proxy-header = { version = "0.1.0", features = ["tokio"] } regex = "1.7.0" blake3 = "1.3.3" lru-cache = "0.1.2" diff --git a/crates/utils/src/acme/cache.rs b/crates/utils/src/acme/cache.rs deleted file mode 100644 index db8fce53..00000000 --- a/crates/utils/src/acme/cache.rs +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{io::ErrorKind, path::PathBuf}; - -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use ring::digest::{Context, SHA512}; - -use super::{AcmeError, AcmeManager}; - -impl AcmeManager { - pub(crate) async fn load_cert(&self) -> Result>, AcmeError> { - self.read_if_exists("cert", self.domains.as_slice()) - .await - .map_err(AcmeError::CertCacheLoad) - } - - pub(crate) async fn store_cert(&self, cert: &[u8]) -> Result<(), AcmeError> { - self.write("cert", self.domains.as_slice(), cert) - .await - .map_err(AcmeError::CertCacheStore) - } - - pub(crate) async fn load_account(&self) -> Result>, AcmeError> { - self.read_if_exists("key", self.contact.as_slice()) - .await - .map_err(AcmeError::AccountCacheLoad) - } - - pub(crate) async fn store_account(&self, account: &[u8]) -> Result<(), AcmeError> { - self.write("key", self.contact.as_slice(), account) - .await - .map_err(AcmeError::AccountCacheStore) - } - - async fn read_if_exists( - &self, - class: &str, - items: &[String], - ) -> Result>, std::io::Error> { - match tokio::fs::read(self.build_filename(class, items)).await { - Ok(content) => Ok(Some(content)), - Err(err) => match err.kind() { - ErrorKind::NotFound => Ok(None), - _ => Err(err), - }, - } - } - - async fn write( - &self, - class: &str, - items: &[String], - contents: impl AsRef<[u8]>, - ) -> Result<(), std::io::Error> { - tokio::fs::create_dir_all(&self.cache_path).await?; - tokio::fs::write(self.build_filename(class, items), contents.as_ref()).await - } - - fn build_filename(&self, class: &str, items: &[String]) -> PathBuf { - let mut ctx = Context::new(&SHA512); - for el in items { - ctx.update(el.as_ref()); - ctx.update(&[0]) - } - ctx.update(self.directory_url.as_bytes()); - - self.cache_path.join(format!( - "{}.{}", - URL_SAFE_NO_PAD.encode(ctx.finish()), - class - )) - } -} diff --git a/crates/utils/src/acme/directory.rs b/crates/utils/src/acme/directory.rs deleted file mode 100644 index fafd3b85..00000000 --- a/crates/utils/src/acme/directory.rs +++ /dev/null @@ -1,365 +0,0 @@ -// Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. - -use std::time::Duration; - -use base64::engine::general_purpose::URL_SAFE_NO_PAD; -use base64::Engine; -use rcgen::{Certificate, CustomExtension, PKCS_ECDSA_P256_SHA256}; -use reqwest::header::{ToStrError, CONTENT_TYPE}; -use reqwest::{Method, Response, StatusCode}; -use ring::error::{KeyRejected, Unspecified}; -use ring::rand::SystemRandom; -use ring::signature::{EcdsaKeyPair, EcdsaSigningAlgorithm, ECDSA_P256_SHA256_FIXED_SIGNING}; -use rustls::crypto::ring::sign::any_ecdsa_type; -use rustls::sign::CertifiedKey; -use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; -use serde::{Deserialize, Serialize}; -use serde_json::json; - -use super::jose::{key_authorization_sha256, sign, JoseError}; - -pub const LETS_ENCRYPT_STAGING_DIRECTORY: &str = - "https://acme-staging-v02.api.letsencrypt.org/directory"; -pub const LETS_ENCRYPT_PRODUCTION_DIRECTORY: &str = - "https://acme-v02.api.letsencrypt.org/directory"; -pub const ACME_TLS_ALPN_NAME: &[u8] = b"acme-tls/1"; - -#[derive(Debug)] -pub struct Account { - pub key_pair: EcdsaKeyPair, - pub directory: Directory, - pub kid: String, -} - -static ALG: &EcdsaSigningAlgorithm = &ECDSA_P256_SHA256_FIXED_SIGNING; - -impl Account { - pub fn generate_key_pair() -> Vec { - EcdsaKeyPair::generate_pkcs8(ALG, &SystemRandom::new()) - .unwrap() - .as_ref() - .to_vec() - } - - pub async fn create<'a, S, I>(directory: Directory, contact: I) -> Result - where - S: AsRef + 'a, - I: IntoIterator, - { - Self::create_with_keypair(directory, contact, &Self::generate_key_pair()).await - } - - pub async fn create_with_keypair<'a, S, I>( - directory: Directory, - contact: I, - key_pair: &[u8], - ) -> Result - where - S: AsRef + 'a, - I: IntoIterator, - { - let key_pair = EcdsaKeyPair::from_pkcs8(ALG, key_pair, &SystemRandom::new())?; - let contact: Vec<&'a str> = contact.into_iter().map(AsRef::::as_ref).collect(); - let payload = json!({ - "termsOfServiceAgreed": true, - "contact": contact, - }) - .to_string(); - let body = sign( - &key_pair, - None, - directory.nonce().await?, - &directory.new_account, - &payload, - )?; - let response = https(&directory.new_account, Method::POST, Some(body)).await?; - let kid = get_header(&response, "Location")?; - Ok(Account { - key_pair, - kid, - directory, - }) - } - - async fn request( - &self, - url: impl AsRef, - payload: &str, - ) -> Result<(Option, String), DirectoryError> { - let body = sign( - &self.key_pair, - Some(&self.kid), - self.directory.nonce().await?, - url.as_ref(), - payload, - )?; - let response = https(url.as_ref(), Method::POST, Some(body)).await?; - let location = get_header(&response, "Location").ok(); - let body = response.text().await?; - Ok((location, body)) - } - - pub async fn new_order(&self, domains: Vec) -> Result<(String, Order), DirectoryError> { - let domains: Vec = domains.into_iter().map(Identifier::Dns).collect(); - let payload = format!("{{\"identifiers\":{}}}", serde_json::to_string(&domains)?); - let response = self.request(&self.directory.new_order, &payload).await?; - let url = response - .0 - .ok_or(DirectoryError::MissingHeader("Location"))?; - let order = serde_json::from_str(&response.1)?; - Ok((url, order)) - } - - pub async fn auth(&self, url: impl AsRef) -> Result { - let response = self.request(url, "").await?; - serde_json::from_str(&response.1).map_err(Into::into) - } - - pub async fn challenge(&self, url: impl AsRef) -> Result<(), DirectoryError> { - self.request(&url, "{}").await.map(|_| ()) - } - - pub async fn order(&self, url: impl AsRef) -> Result { - let response = self.request(&url, "").await?; - serde_json::from_str(&response.1).map_err(Into::into) - } - - pub async fn finalize( - &self, - url: impl AsRef, - csr: Vec, - ) -> Result { - let payload = format!("{{\"csr\":\"{}\"}}", URL_SAFE_NO_PAD.encode(csr)); - let response = self.request(&url, &payload).await?; - serde_json::from_str(&response.1).map_err(Into::into) - } - - pub async fn certificate(&self, url: impl AsRef) -> Result { - Ok(self.request(&url, "").await?.1) - } - - pub fn tls_alpn_01<'a>( - &self, - challenges: &'a [Challenge], - domain: String, - ) -> Result<(&'a Challenge, CertifiedKey), DirectoryError> { - let challenge = challenges - .iter() - .find(|c| c.typ == ChallengeType::TlsAlpn01); - let challenge = match challenge { - Some(challenge) => challenge, - None => return Err(DirectoryError::NoTlsAlpn01Challenge), - }; - let mut params = rcgen::CertificateParams::new(vec![domain]); - let key_auth = key_authorization_sha256(&self.key_pair, &challenge.token)?; - params.alg = &PKCS_ECDSA_P256_SHA256; - params.custom_extensions = vec![CustomExtension::new_acme_identifier(key_auth.as_ref())]; - let cert = Certificate::from_params(params)?; - let pk = any_ecdsa_type(&PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( - cert.serialize_private_key_der(), - ))) - .unwrap(); - let certified_key = - CertifiedKey::new(vec![CertificateDer::from(cert.serialize_der()?)], pk); - Ok((challenge, certified_key)) - } -} - -#[derive(Debug, Clone, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Directory { - pub new_nonce: String, - pub new_account: String, - pub new_order: String, -} - -impl Directory { - pub async fn discover(url: impl AsRef) -> Result { - Ok(serde_json::from_str( - &https(url, Method::GET, None).await?.text().await?, - )?) - } - pub async fn nonce(&self) -> Result { - get_header( - &https(&self.new_nonce.as_str(), Method::HEAD, None).await?, - "replay-nonce", - ) - } -} - -#[derive(Debug, Deserialize, Eq, PartialEq)] -pub enum ChallengeType { - #[serde(rename = "http-01")] - Http01, - #[serde(rename = "dns-01")] - Dns01, - #[serde(rename = "tls-alpn-01")] - TlsAlpn01, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Order { - #[serde(flatten)] - pub status: OrderStatus, - pub authorizations: Vec, - pub finalize: String, - pub error: Option, -} - -#[derive(Debug, Deserialize, Clone, PartialEq, Eq)] -#[serde(tag = "status", rename_all = "camelCase")] -pub enum OrderStatus { - Pending, - Ready, - Valid { certificate: String }, - Invalid, - Processing, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Auth { - pub status: AuthStatus, - pub identifier: Identifier, - pub challenges: Vec, -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum AuthStatus { - Pending, - Valid, - Invalid, - Revoked, - Expired, - Deactivated, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(tag = "type", content = "value", rename_all = "camelCase")] -pub enum Identifier { - Dns(String), -} - -#[derive(Debug, Deserialize)] -pub struct Challenge { - #[serde(rename = "type")] - pub typ: ChallengeType, - pub url: String, - pub token: String, - pub error: Option, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct Problem { - #[serde(rename = "type")] - pub typ: Option, - pub detail: Option, -} - -#[derive(Debug)] -pub enum DirectoryError { - Io(std::io::Error), - Rcgen(rcgen::Error), - Jose(JoseError), - Json(serde_json::Error), - HttpRequest(reqwest::Error), - HttpRequestCode { code: StatusCode, reason: String }, - HttpResponseNonStringHeader(ToStrError), - KeyRejected(KeyRejected), - Crypto(Unspecified), - MissingHeader(&'static str), - NoTlsAlpn01Challenge, -} - -#[allow(unused_mut)] -async fn https( - url: impl AsRef, - method: Method, - body: Option, -) -> Result { - let url = url.as_ref(); - let mut builder = reqwest::Client::builder().timeout(Duration::from_secs(30)); - - #[cfg(debug_assertions)] - { - builder = builder.danger_accept_invalid_certs( - url.starts_with("https://localhost") || url.starts_with("https://127.0.0.1"), - ); - } - - let mut request = builder.build()?.request(method, url); - - if let Some(body) = body { - request = request - .header(CONTENT_TYPE, "application/jose+json") - .body(body); - } - - let response = request.send().await?; - if response.status().is_success() { - Ok(response) - } else { - Err(DirectoryError::HttpRequestCode { - code: response.status(), - reason: response.text().await?, - }) - } -} - -fn get_header(response: &Response, header: &'static str) -> Result { - match response.headers().get_all(header).iter().last() { - Some(value) => Ok(value.to_str()?.to_string()), - None => Err(DirectoryError::MissingHeader(header)), - } -} - -impl From for DirectoryError { - fn from(err: std::io::Error) -> Self { - Self::Io(err) - } -} - -impl From for DirectoryError { - fn from(err: rcgen::Error) -> Self { - Self::Rcgen(err) - } -} - -impl From for DirectoryError { - fn from(err: JoseError) -> Self { - Self::Jose(err) - } -} - -impl From for DirectoryError { - fn from(err: serde_json::Error) -> Self { - Self::Json(err) - } -} - -impl From for DirectoryError { - fn from(err: reqwest::Error) -> Self { - Self::HttpRequest(err) - } -} - -impl From for DirectoryError { - fn from(err: KeyRejected) -> Self { - Self::KeyRejected(err) - } -} - -impl From for DirectoryError { - fn from(err: Unspecified) -> Self { - Self::Crypto(err) - } -} - -impl From for DirectoryError { - fn from(err: ToStrError) -> Self { - Self::HttpResponseNonStringHeader(err) - } -} diff --git a/crates/utils/src/acme/jose.rs b/crates/utils/src/acme/jose.rs deleted file mode 100644 index f8eb7472..00000000 --- a/crates/utils/src/acme/jose.rs +++ /dev/null @@ -1,140 +0,0 @@ -// Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. - -use base64::engine::general_purpose::URL_SAFE_NO_PAD; -use base64::Engine; -use ring::digest::{digest, Digest, SHA256}; -use ring::rand::SystemRandom; -use ring::signature::{EcdsaKeyPair, KeyPair}; -use serde::Serialize; - -pub(crate) fn sign( - key: &EcdsaKeyPair, - kid: Option<&str>, - nonce: String, - url: &str, - payload: &str, -) -> Result { - let jwk = match kid { - None => Some(Jwk::new(key)), - Some(_) => None, - }; - let protected = Protected::base64(jwk, kid, nonce, url)?; - let payload = URL_SAFE_NO_PAD.encode(payload); - let combined = format!("{}.{}", &protected, &payload); - let signature = key.sign(&SystemRandom::new(), combined.as_bytes())?; - let signature = URL_SAFE_NO_PAD.encode(signature.as_ref()); - let body = Body { - protected, - payload, - signature, - }; - Ok(serde_json::to_string(&body)?) -} - -pub(crate) fn key_authorization_sha256( - key: &EcdsaKeyPair, - token: &str, -) -> Result { - let jwk = Jwk::new(key); - let key_authorization = format!("{}.{}", token, jwk.thumb_sha256_base64()?); - Ok(digest(&SHA256, key_authorization.as_bytes())) -} - -#[derive(Serialize)] -struct Body { - protected: String, - payload: String, - signature: String, -} - -#[derive(Serialize)] -struct Protected<'a> { - alg: &'static str, - #[serde(skip_serializing_if = "Option::is_none")] - jwk: Option, - #[serde(skip_serializing_if = "Option::is_none")] - kid: Option<&'a str>, - nonce: String, - url: &'a str, -} - -impl<'a> Protected<'a> { - fn base64( - jwk: Option, - kid: Option<&'a str>, - nonce: String, - url: &'a str, - ) -> Result { - let protected = Self { - alg: "ES256", - jwk, - kid, - nonce, - url, - }; - let protected = serde_json::to_vec(&protected)?; - Ok(URL_SAFE_NO_PAD.encode(protected)) - } -} - -#[derive(Serialize)] -struct Jwk { - alg: &'static str, - crv: &'static str, - kty: &'static str, - #[serde(rename = "use")] - u: &'static str, - x: String, - y: String, -} - -impl Jwk { - pub(crate) fn new(key: &EcdsaKeyPair) -> Self { - let (x, y) = key.public_key().as_ref()[1..].split_at(32); - Self { - alg: "ES256", - crv: "P-256", - kty: "EC", - u: "sig", - x: URL_SAFE_NO_PAD.encode(x), - y: URL_SAFE_NO_PAD.encode(y), - } - } - pub(crate) fn thumb_sha256_base64(&self) -> Result { - let jwk_thumb = JwkThumb { - crv: self.crv, - kty: self.kty, - x: &self.x, - y: &self.y, - }; - let json = serde_json::to_vec(&jwk_thumb)?; - let hash = digest(&SHA256, &json); - Ok(URL_SAFE_NO_PAD.encode(hash)) - } -} - -#[derive(Serialize)] -struct JwkThumb<'a> { - crv: &'a str, - kty: &'a str, - x: &'a str, - y: &'a str, -} - -#[derive(Debug)] -pub enum JoseError { - Json(serde_json::Error), - Crypto(ring::error::Unspecified), -} - -impl From for JoseError { - fn from(err: serde_json::Error) -> Self { - Self::Json(err) - } -} - -impl From for JoseError { - fn from(err: ring::error::Unspecified) -> Self { - Self::Crypto(err) - } -} diff --git a/crates/utils/src/acme/mod.rs b/crates/utils/src/acme/mod.rs deleted file mode 100644 index a05e23a5..00000000 --- a/crates/utils/src/acme/mod.rs +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -pub mod cache; -pub mod directory; -pub mod jose; -pub mod order; -pub mod resolver; - -use std::{ - fmt::Debug, - path::PathBuf, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, - time::Duration, -}; - -use ahash::AHashMap; -use arc_swap::ArcSwap; -use parking_lot::Mutex; -use rustls::sign::CertifiedKey; -use tokio::sync::watch; - -use crate::config::tls::build_self_signed_cert; - -use self::{ - directory::Account, - order::{CertParseError, OrderError}, -}; - -pub struct AcmeManager { - pub(crate) directory_url: String, - pub(crate) domains: Vec, - contact: Vec, - renew_before: chrono::Duration, - cache_path: PathBuf, - account_key: ArcSwap>, - auth_keys: Mutex>>, - order_in_progress: AtomicBool, - cert: ArcSwap, -} - -#[derive(Debug)] -pub enum AcmeError { - CertCacheLoad(std::io::Error), - AccountCacheLoad(std::io::Error), - CertCacheStore(std::io::Error), - AccountCacheStore(std::io::Error), - CachedCertParse(CertParseError), - Order(OrderError), - NewCertParse(CertParseError), -} - -impl AcmeManager { - pub fn new( - directory_url: String, - domains: Vec, - contact: Vec, - renew_before: Duration, - cache_path: PathBuf, - ) -> crate::config::Result { - Ok(AcmeManager { - directory_url, - contact: contact - .into_iter() - .map(|c| { - if !c.starts_with("mailto:") { - format!("mailto:{}", c) - } else { - c - } - }) - .collect(), - renew_before: chrono::Duration::from_std(renew_before).unwrap(), - cache_path, - account_key: ArcSwap::from_pointee(Vec::new()), - auth_keys: Mutex::new(AHashMap::new()), - order_in_progress: false.into(), - cert: ArcSwap::from_pointee(build_self_signed_cert(&domains)?), - domains, - }) - } - - pub async fn init(&self) -> Result { - // Load account key from cache or generate a new one - if let Some(account_key) = self.load_account().await? { - self.account_key.store(Arc::new(account_key)); - } else { - let account_key = Account::generate_key_pair(); - self.store_account(&account_key).await?; - self.account_key.store(Arc::new(account_key)); - } - - // Load certificate from cache or request a new one - Ok(if let Some(pem) = self.load_cert().await? { - self.process_cert(pem, true).await? - } else { - Duration::from_millis(1000) - }) - } - - pub fn has_order_in_progress(&self) -> bool { - self.order_in_progress.load(Ordering::Relaxed) - } -} - -pub trait SpawnAcme { - fn spawn(self, shutdown_rx: watch::Receiver); -} - -impl SpawnAcme for Arc { - fn spawn(self, mut shutdown_rx: watch::Receiver) { - tokio::spawn(async move { - let acme = self; - let mut renew_at = match acme.init().await { - Ok(renew_at) => renew_at, - Err(err) => { - tracing::error!( - context = "acme", - event = "error", - error = ?err, - "Failed to initialize ACME certificate manager."); - - return; - } - }; - - loop { - tokio::select! { - _ = tokio::time::sleep(renew_at) => { - tracing::info!( - context = "acme", - event = "order", - domains = ?acme.domains, - "Ordering certificates."); - - match acme.renew().await { - Ok(renew_at_) => { - renew_at = renew_at_; - tracing::info!( - context = "acme", - event = "success", - domains = ?acme.domains, - next_renewal = ?renew_at, - "Certificates renewed."); - }, - Err(err) => { - tracing::error!( - context = "acme", - event = "error", - error = ?err, - "Failed to renew certificates."); - - renew_at = Duration::from_secs(3600); - }, - } - - }, - _ = shutdown_rx.changed() => { - tracing::debug!( - context = "acme", - event = "shutdown", - domains = ?acme.domains, - "ACME certificate manager shutting down."); - - break; - } - }; - } - }); - } -} - -impl Debug for AcmeManager { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AcmeManager") - .field("directory_url", &self.directory_url) - .field("domains", &self.domains) - .field("contact", &self.contact) - .field("cache_path", &self.cache_path) - .field("account_key", &self.account_key) - .finish() - } -} diff --git a/crates/utils/src/acme/order.rs b/crates/utils/src/acme/order.rs deleted file mode 100644 index c72eacec..00000000 --- a/crates/utils/src/acme/order.rs +++ /dev/null @@ -1,300 +0,0 @@ -// Adapted from rustls-acme (https://github.com/FlorianUekermann/rustls-acme), licensed under MIT/Apache-2.0. - -use chrono::{DateTime, TimeZone, Utc}; -use futures::future::try_join_all; -use rcgen::{CertificateParams, DistinguishedName, PKCS_ECDSA_P256_SHA256}; -use rustls::crypto::ring::sign::any_ecdsa_type; -use rustls::sign::CertifiedKey; -use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; -use std::fmt::Debug; -use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::time::Duration; -use x509_parser::parse_x509_certificate; - -use crate::acme::directory::Identifier; - -use super::directory::{Account, Auth, AuthStatus, Directory, DirectoryError, Order, OrderStatus}; -use super::jose::JoseError; -use super::{AcmeError, AcmeManager}; - -#[derive(Debug)] -pub enum OrderError { - Acme(DirectoryError), - Rcgen(rcgen::Error), - BadOrder(Order), - BadAuth(Auth), - TooManyAttemptsAuth(String), - ProcessingTimeout(Order), -} - -#[derive(Debug)] -pub enum CertParseError { - X509(x509_parser::nom::Err), - Pem(pem::PemError), - TooFewPem(usize), - InvalidPrivateKey, -} - -impl AcmeManager { - pub(crate) async fn process_cert( - &self, - pem: Vec, - cached: bool, - ) -> Result { - let (cert, validity) = match (parse_cert(&pem), cached) { - (Ok(r), _) => r, - (Err(err), cached) => { - return match cached { - true => Err(AcmeError::CachedCertParse(err)), - false => Err(AcmeError::NewCertParse(err)), - } - } - }; - - self.set_cert(Arc::new(cert)); - - let renew_at = (validity[1] - self.renew_before - Utc::now()) - .max(chrono::Duration::zero()) - .to_std() - .unwrap_or_default(); - let renewal_date = validity[1] - self.renew_before; - - tracing::info!( - context = "acme", - event = "process-cert", - valid_not_before = %validity[0], - valid_not_after = %validity[1], - renewal_date = ?renewal_date, - domains = ?self.domains, - "Loaded certificate for domains {:?}", self.domains); - - if !cached { - self.store_cert(&pem).await?; - } - - Ok(renew_at) - } - - pub async fn renew(&self) -> Result { - let mut backoff = 0; - self.order_in_progress.store(true, Ordering::Relaxed); - loop { - match self.order().await { - Ok(pem) => return self.process_cert(pem, false).await, - Err(err) if backoff < 16 => { - tracing::debug!( - context = "acme", - event = "renew-backoff", - domains = ?self.domains, - attempt = backoff, - reason = ?err, - "Failed to renew certificate, backing off for {} seconds", - 1 << backoff); - backoff = (backoff + 1).min(16); - tokio::time::sleep(Duration::from_secs(1 << backoff)).await; - } - Err(err) => return Err(AcmeError::Order(err)), - } - } - } - - async fn order(&self) -> Result, OrderError> { - let directory = Directory::discover(&self.directory_url).await?; - let account = Account::create_with_keypair( - directory, - &self.contact, - self.account_key.load().as_slice(), - ) - .await?; - - let mut params = CertificateParams::new(self.domains.clone()); - params.distinguished_name = DistinguishedName::new(); - params.alg = &PKCS_ECDSA_P256_SHA256; - let cert = rcgen::Certificate::from_params(params)?; - - let (order_url, mut order) = account.new_order(self.domains.clone()).await?; - loop { - match order.status { - OrderStatus::Pending => { - let auth_futures = order - .authorizations - .iter() - .map(|url| self.authorize(&account, url)); - try_join_all(auth_futures).await?; - tracing::info!( - context = "acme", - event = "auth-complete", - domains = ?self.domains.as_slice(), - "Completed all authorizations" - ); - order = account.order(&order_url).await?; - } - OrderStatus::Processing => { - for i in 0u64..10 { - tracing::info!( - context = "acme", - event = "processing", - domains = ?self.domains.as_slice(), - attempt = i, - "Processing order" - ); - tokio::time::sleep(Duration::from_secs(1u64 << i)).await; - order = account.order(&order_url).await?; - if order.status != OrderStatus::Processing { - break; - } - } - if order.status == OrderStatus::Processing { - return Err(OrderError::ProcessingTimeout(order)); - } - } - OrderStatus::Ready => { - tracing::info!( - context = "acme", - event = "csr-send", - domains = ?self.domains.as_slice(), - "Sending CSR" - ); - - let csr = cert.serialize_request_der()?; - order = account.finalize(order.finalize, csr).await? - } - OrderStatus::Valid { certificate } => { - tracing::info!( - context = "acme", - event = "download", - domains = ?self.domains.as_slice(), - "Downloading certificate" - ); - - let pem = [ - &cert.serialize_private_key_pem(), - "\n", - &account.certificate(certificate).await?, - ] - .concat(); - return Ok(pem.into_bytes()); - } - OrderStatus::Invalid => { - tracing::warn!( - context = "acme", - event = "error", - reason = "invalid-order", - domains = ?self.domains.as_slice(), - "Invalid order" - ); - - return Err(OrderError::BadOrder(order)); - } - } - } - } - - async fn authorize(&self, account: &Account, url: &String) -> Result<(), OrderError> { - let auth = account.auth(url).await?; - let (domain, challenge_url) = match auth.status { - AuthStatus::Pending => { - let Identifier::Dns(domain) = auth.identifier; - tracing::info!( - context = "acme", - event = "challenge", - domain = domain, - "Requesting challenge for domain {domain}" - ); - let (challenge, auth_key) = - account.tls_alpn_01(&auth.challenges, domain.clone())?; - self.set_auth_key(domain.clone(), Arc::new(auth_key)); - account.challenge(&challenge.url).await?; - (domain, challenge.url.clone()) - } - AuthStatus::Valid => return Ok(()), - _ => return Err(OrderError::BadAuth(auth)), - }; - for i in 0u64..5 { - tokio::time::sleep(Duration::from_secs(1u64 << i)).await; - let auth = account.auth(url).await?; - match auth.status { - AuthStatus::Pending => { - tracing::info!( - context = "acme", - event = "auth-pending", - domain = domain, - attempt = i, - "Authorization for domain {domain} is still pending", - ); - account.challenge(&challenge_url).await? - } - AuthStatus::Valid => return Ok(()), - _ => return Err(OrderError::BadAuth(auth)), - } - } - Err(OrderError::TooManyAttemptsAuth(domain)) - } -} - -fn parse_cert(pem: &[u8]) -> Result<(CertifiedKey, [DateTime; 2]), CertParseError> { - let mut pems = pem::parse_many(pem)?; - if pems.len() < 2 { - return Err(CertParseError::TooFewPem(pems.len())); - } - let pk = match any_ecdsa_type(&PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( - pems.remove(0).contents(), - ))) { - Ok(pk) => pk, - Err(_) => return Err(CertParseError::InvalidPrivateKey), - }; - let cert_chain: Vec = pems - .into_iter() - .map(|p| CertificateDer::from(p.into_contents())) - .collect(); - let validity = match parse_x509_certificate(&cert_chain[0]) { - Ok((_, cert)) => { - let validity = cert.validity(); - [validity.not_before, validity.not_after].map(|t| { - Utc.timestamp_opt(t.timestamp(), 0) - .earliest() - .unwrap_or_default() - }) - } - Err(err) => return Err(CertParseError::X509(err)), - }; - let cert = CertifiedKey::new(cert_chain, pk); - Ok((cert, validity)) -} - -impl From for OrderError { - fn from(err: DirectoryError) -> Self { - Self::Acme(err) - } -} - -impl From for OrderError { - fn from(err: rcgen::Error) -> Self { - Self::Rcgen(err) - } -} - -impl From> for CertParseError { - fn from(err: x509_parser::nom::Err) -> Self { - Self::X509(err) - } -} - -impl From for CertParseError { - fn from(err: pem::PemError) -> Self { - Self::Pem(err) - } -} - -impl From for OrderError { - fn from(err: JoseError) -> Self { - Self::Acme(DirectoryError::Jose(err)) - } -} - -impl From for AcmeError { - fn from(err: JoseError) -> Self { - Self::Order(OrderError::from(err)) - } -} diff --git a/crates/utils/src/acme/resolver.rs b/crates/utils/src/acme/resolver.rs deleted file mode 100644 index 58da89ce..00000000 --- a/crates/utils/src/acme/resolver.rs +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::sync::{atomic::Ordering, Arc}; - -use rustls::{ - server::{ClientHello, ResolvesServerCert}, - sign::CertifiedKey, -}; - -use super::{directory::ACME_TLS_ALPN_NAME, AcmeManager}; - -impl AcmeManager { - pub(crate) fn set_cert(&self, cert: Arc) { - self.cert.store(cert); - self.order_in_progress.store(false, Ordering::Relaxed); - self.auth_keys.lock().clear(); - } - pub(crate) fn set_auth_key(&self, domain: String, cert: Arc) { - self.auth_keys.lock().insert(domain, cert); - } -} - -impl ResolvesServerCert for AcmeManager { - fn resolve(&self, client_hello: ClientHello) -> Option> { - if self.has_order_in_progress() && client_hello.is_tls_alpn_challenge() { - match client_hello.server_name() { - None => { - tracing::debug!( - context = "acme", - event = "error", - reason = "missing-sni", - "client did not supply SNI" - ); - None - } - Some(domain) => { - tracing::trace!( - context = "acme", - event = "auth-key", - domain = %domain, - "Found client supplied SNI"); - - self.auth_keys.lock().get(domain).cloned() - } - } - } else { - self.cert.load().clone().into() - } - } -} - -pub trait IsTlsAlpnChallenge { - fn is_tls_alpn_challenge(&self) -> bool; -} - -impl IsTlsAlpnChallenge for ClientHello<'_> { - fn is_tls_alpn_challenge(&self) -> bool { - self.alpn().into_iter().flatten().eq([ACME_TLS_ALPN_NAME]) - } -} diff --git a/crates/utils/src/config/if_block.rs b/crates/utils/src/config/if_block.rs deleted file mode 100644 index 6ad29ca2..00000000 --- a/crates/utils/src/config/if_block.rs +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use crate::expr::{Constant, Expression, Token, Variable}; - -use super::{utils::AsKey, Config}; - -#[derive(Debug, Clone, Default)] -#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] -pub struct IfThen { - pub expr: Expression, - pub then: Expression, -} - -#[derive(Debug, Clone, Default)] -#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] -pub struct IfBlock { - pub key: String, - pub if_then: Vec, - pub default: Expression, -} - -impl IfBlock { - pub fn new>(value: T) -> Self { - Self { - key: String::new(), - if_then: Vec::new(), - default: Expression::from(value), - } - } - - pub async fn eval<'x, V, F, R>(&'x self, var: V, mut fnc: F) -> Variable<'x> - where - V: Fn(u32) -> Variable<'x>, - F: FnMut(u32, Vec>) -> R, - R: std::future::Future> + Send, - { - let mut captures = Vec::new(); - - for if_then in &self.if_then { - if if_then - .expr - .eval(&var, &mut fnc, &mut captures) - .await - .to_bool() - { - return if_then.then.eval(&var, &mut fnc, &mut captures).await; - } - } - - self.default.eval(&var, &mut fnc, &mut captures).await - } - - pub fn is_empty(&self) -> bool { - self.default.is_empty() && self.if_then.is_empty() - } -} - -impl Config { - pub fn parse_if_block( - &self, - prefix: impl AsKey, - token_map: impl Fn(&str) -> Result, - ) -> super::Result> { - let key = prefix.as_key(); - let prefix = prefix.as_prefix(); - - let mut found_if = false; - let mut found_else = ""; - let mut found_then = false; - - // Parse conditions - let mut if_block = IfBlock { - key, - ..Default::default() - }; - let mut last_array_pos = ""; - let key = &if_block.key; - - for (item, value) in &self.keys { - if let Some(suffix_) = item.strip_prefix(&prefix) { - if let Some((array_pos, suffix)) = suffix_.split_once('.') { - let if_key = suffix.split_once('.').map(|(v, _)| v).unwrap_or(suffix); - if if_key == "if" { - if array_pos != last_array_pos { - if !last_array_pos.is_empty() && !found_then { - return Err(format!( - "Missing 'then' in 'if' condition {} for property {:?}.", - last_array_pos.parse().unwrap_or(0) + 1, - key - )); - } - - if_block.if_then.push(IfThen { - expr: Expression::parse(key.as_str(), value, &token_map)?, - then: Expression::default(), - }); - - found_then = false; - last_array_pos = array_pos; - } - - found_if = true; - } else if if_key == "else" { - if found_else.is_empty() { - if found_if { - if_block.default = - Expression::parse(key.as_str(), value, &token_map)?; - found_else = array_pos; - } else { - return Err(format!( - "Found 'else' before 'if' for property {key:?}.", - )); - } - } else if array_pos != found_else { - return Err(format!("Multiple 'else' found for property {key:?}.")); - } - } else if if_key == "then" { - if found_else.is_empty() { - if array_pos == last_array_pos { - if !found_then { - if_block.if_then.last_mut().unwrap().then = - Expression::parse(key.as_str(), value, &token_map)?; - found_then = true; - } - } else { - return Err(format!( - "Found 'then' without 'if' for property {key:?}.", - )); - } - } else { - return Err(format!( - "Found 'then' in 'else' block for property {key:?}.", - )); - } - } - } else { - return Err(format!("Invalid property {item:?} found in 'if' block.")); - } - } else if item == key { - // There is a single value, parse and return - if_block.default = Expression::parse(key.as_str(), value, &token_map)?; - return Ok(Some(if_block)); - } - } - - if !found_if { - Ok(None) - } else if !found_then { - Err(format!( - "Missing 'then' in 'if' condition {} for property {:?}.", - last_array_pos.parse().unwrap_or(0) + 1, - key - )) - } else if found_else.is_empty() { - Err(format!("Missing 'else' for property {key:?}.")) - } else { - Ok(Some(if_block)) - } - } -} diff --git a/crates/utils/src/config/ipmask.rs b/crates/utils/src/config/ipmask.rs index 70efba3d..4afaa97f 100644 --- a/crates/utils/src/config/ipmask.rs +++ b/crates/utils/src/config/ipmask.rs @@ -21,7 +21,9 @@ * for more details. */ -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; + +use rustls::{crypto::ring::cipher_suite::*, SupportedCipherSuite}; use super::utils::{AsKey, ParseKey, ParseValue}; @@ -146,6 +148,47 @@ impl ParseValue for IpAddrOrMask { } } +impl ParseValue for SocketAddr { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + value.parse().map_err(|_| { + format!( + "Invalid socket address {:?} for property {:?}.", + value, + key.as_key() + ) + }) + } +} + +impl ParseValue for SupportedCipherSuite { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + Ok(match value { + // TLS1.3 suites + "TLS13_AES_256_GCM_SHA384" => TLS13_AES_256_GCM_SHA384, + "TLS13_AES_128_GCM_SHA256" => TLS13_AES_128_GCM_SHA256, + "TLS13_CHACHA20_POLY1305_SHA256" => TLS13_CHACHA20_POLY1305_SHA256, + // TLS1.2 suites + "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" => TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" => TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" => { + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 + } + "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" => TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" => TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" => { + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 + } + cipher => { + return Err(format!( + "Unsupported TLS cipher suite {:?} found in key {:?}", + cipher, + key.as_key() + )) + } + }) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/utils/src/config/listener.rs b/crates/utils/src/config/listener.rs deleted file mode 100644 index e212e8e8..00000000 --- a/crates/utils/src/config/listener.rs +++ /dev/null @@ -1,444 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{net::SocketAddr, sync::Arc}; - -use ahash::AHashMap; -use rustls::{ - crypto::ring::{ - cipher_suite::{ - TLS13_AES_128_GCM_SHA256, TLS13_AES_256_GCM_SHA384, TLS13_CHACHA20_POLY1305_SHA256, - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, - }, - default_provider, - }, - server::ResolvesServerCert, - ServerConfig, SupportedCipherSuite, ALL_VERSIONS, -}; -use tokio::net::TcpSocket; -use tokio_rustls::TlsAcceptor; - -use crate::{ - acme::{directory::ACME_TLS_ALPN_NAME, AcmeManager}, - listener::{ - tls::{Certificate, CertificateResolver}, - TcpAcceptor, - }, - UnwrapFailure, -}; - -use super::{ - tls::{TLS12_VERSION, TLS13_VERSION}, - utils::{AsKey, ParseKey, ParseValue}, - Config, Listener, Server, ServerProtocol, Servers, -}; - -impl Config { - pub fn parse_servers(&self) -> super::Result { - let mut servers = Servers::default(); - - // Parse certificates and ACME managers - let certificates = self.parse_certificates()?; - let acmes = self.parse_acmes()?; - - // Parse servers - for (internal_id, id) in self.sub_keys("server.listener", ".protocol").enumerate() { - let mut server = self.parse_server(id, &certificates, &acmes)?; - if !servers.inner.iter().any(|s| s.id == server.id) { - server.internal_id = internal_id as u16; - servers.inner.push(server); - } else { - return Err(format!("Duplicate listener id {:?}.", server.id)); - } - } - - // Add certificates with valid paths - for (id, cert) in certificates { - if cert.path.len() == 2 { - servers.certificates.push(cert); - } else { - tracing::debug!( - context = "config", - event = "acme", - id = id, - "Certificate reloading disabled for id {id:?}", - ); - } - } - - // Add ACME managers with configured domains - for (id, acme) in acmes { - if !acme.domains.is_empty() { - servers.acme_managers.push(acme); - } else { - tracing::debug!( - context = "config", - event = "acme", - id = id, - "ACME certificate manager disabled for id {id:?}", - ); - } - } - - if !servers.inner.is_empty() { - Ok(servers) - } else { - Err("No server directives found in config file.".to_string()) - } - } - - fn parse_server( - &self, - id: &str, - certificates: &AHashMap>, - acmes: &AHashMap>, - ) -> super::Result { - // Build listeners - let mut listeners = Vec::new(); - for result in self.properties::(("server.listener", id, "bind")) { - // Parse bind address and build socket - let (_, addr) = result?; - let socket = if addr.is_ipv4() { - TcpSocket::new_v4() - } else { - TcpSocket::new_v6() - } - .map_err(|err| format!("Failed to create socket: {err}"))?; - - // Set socket options - for option in [ - "reuse-addr", - "reuse-port", - "send-buffer-size", - "recv-buffer-size", - "tos", - ] { - if let Some(value) = self.value_or_else( - ("server.listener", id, "socket", option), - ("server.socket", option), - ) { - let key = ("server.listener", id, "socket", option); - match option { - "reuse-addr" => socket.set_reuseaddr(value.parse_key(key)?), - #[cfg(not(target_env = "msvc"))] - "reuse-port" => socket.set_reuseport(value.parse_key(key)?), - "send-buffer-size" => socket.set_send_buffer_size(value.parse_key(key)?), - "recv-buffer-size" => socket.set_recv_buffer_size(value.parse_key(key)?), - "tos" => socket.set_tos(value.parse_key(key)?), - _ => continue, - } - .map_err(|err| { - format!("Failed to set socket option '{option}' for listener '{id}': {err}") - })?; - } - } - - listeners.push(Listener { - socket, - addr, - ttl: self - .property_or_else(("server.listener", id, "socket.ttl"), "server.socket.ttl")?, - backlog: self.property_or_else( - ("server.listener", id, "socket.backlog"), - "server.socket.backlog", - )?, - linger: self.property_or_else( - ("server.listener", id, "socket.linger"), - "server.socket.linger", - )?, - nodelay: self - .property_or_else( - ("server.listener", id, "socket.nodelay"), - "server.socket.nodelay", - )? - .unwrap_or(true), - }); - } - - if listeners.is_empty() { - return Err(format!("No 'bind' directive found for listener id {id:?}")); - } - - // Build TLS config - let (acceptor, tls_implicit) = if self - .property_or_else(("server.listener", id, "tls.enable"), "server.tls.enable")? - .unwrap_or(false) - { - // Parse protocol versions - let mut tls_v2 = false; - let mut tls_v3 = false; - for (key, protocol) in self.values_or_else( - ("server.listener", id, "tls.protocols"), - "server.tls.protocols", - ) { - match protocol { - "TLSv1.2" | "0x0303" => tls_v2 = true, - "TLSv1.3" | "0x0304" => tls_v3 = true, - protocol => { - return Err(format!( - "Unsupported TLS protocol {protocol:?} found in key {key:?}", - )) - } - } - } - - // Parse cipher suites - let mut ciphers: Vec = Vec::new(); - for (key, protocol) in - self.values_or_else(("server.listener", id, "tls.ciphers"), "server.tls.ciphers") - { - ciphers.push(protocol.parse_key(key)?); - } - - // Build resolver - let mut acme_acceptor = None; - let resolver: Arc = if let Some(acme_id) = - self.value_or_else(("server.listener", id, "tls.acme"), "server.tls.acme") - { - let acme = acmes.get(acme_id).ok_or_else(|| { - format!("Undefined ACME id {acme_id:?} for listener {id:?}.",) - })?; - - // Check if this port is used to receive ACME challenges - let acme_port = - self.property_or_default::(("acme", acme_id, "port"), "443")?; - if listeners.iter().any(|l| l.addr.port() == acme_port) { - acme_acceptor = Some(acme.clone()); - } - - acme.clone() - } else { - let cert_id = self - .value_or_else( - ("server.listener", id, "tls.certificate"), - "server.tls.certificate", - ) - .ok_or_else(|| format!("Undefined certificate id for listener {id:?}."))?; - let mut resolver = CertificateResolver { - sni: Default::default(), - cert: certificates - .get(cert_id) - .ok_or_else(|| { - format!("Undefined certificate id {cert_id:?} for listener {id:?}.",) - })? - .clone(), - }; - - // Add SNI certificates - for (key, value) in - self.values_or_else(("server.listener", id, "tls.sni"), "server.tls.sni") - { - if let Some(prefix) = key.strip_suffix(".subject") { - resolver - .add( - value, - match self.value((prefix, "certificate")) { - Some(sni_cert_id) if sni_cert_id != cert_id => { - certificates.get(sni_cert_id).ok_or_else(|| { - format!( - "Undefined certificate id {sni_cert_id:?} for SNI {value:?} in listener {id:?}.", - ) - })?.clone() - } - _ => resolver.cert.clone(), - }, - ) - .map_err(|err| { - format!("Failed to add SNI certificate for {key:?}: {err}") - })?; - } - } - - Arc::new(resolver) - }; - - // Build cert provider - let mut provider = default_provider(); - if !ciphers.is_empty() { - provider.cipher_suites = ciphers; - } - - // Build server config - let mut config = ServerConfig::builder_with_provider(provider.into()) - .with_protocol_versions(if tls_v3 == tls_v2 { - ALL_VERSIONS - } else if tls_v3 { - TLS13_VERSION - } else { - TLS12_VERSION - }) - .map_err(|err| format!("Failed to build TLS config: {err}"))? - .with_no_client_auth() - .with_cert_resolver(resolver.clone()); - config.ignore_client_order = self - .property_or_else( - ("server.listener", id, "tls.ignore-client-order"), - "server.tls.ignore-client-order", - )? - .unwrap_or(true); - - // Build acceptor - let acceptor = if let Some(manager) = acme_acceptor { - let mut challenge = ServerConfig::builder() - .with_no_client_auth() - .with_cert_resolver(resolver); - challenge.alpn_protocols.push(ACME_TLS_ALPN_NAME.to_vec()); - TcpAcceptor::Acme { - challenge: Arc::new(challenge), - default: Arc::new(config), - manager, - } - } else { - TcpAcceptor::Tls(TlsAcceptor::from(Arc::new(config))) - }; - - ( - acceptor, - self.property_or_else( - ("server.listener", id, "tls.implicit"), - "server.tls.implicit", - )? - .unwrap_or(true), - ) - } else { - (TcpAcceptor::Plain, false) - }; - - let protocol = self.property_require(("server.listener", id, "protocol"))?; - - // Parse proxy networks - let mut proxy_networks = Vec::new(); - for network in self.set_values_or_else( - ("server.listener", id, "proxy.trusted-networks"), - "server.proxy.trusted-networks", - ) { - proxy_networks.push(network.parse_key("server.proxy.trusted-networks")?); - } - - Ok(Server { - id: id.to_string(), - internal_id: 0, - hostname: self - .value_or_else(("server.listener", id, "hostname"), "server.hostname") - .ok_or("Hostname directive not found.")? - .to_string(), - data: match protocol { - ServerProtocol::Smtp | ServerProtocol::Lmtp => self - .value_or_else(("server.listener", id, "greeting"), "server.greeting") - .unwrap_or(concat!( - "Stalwart SMTP v", - env!("CARGO_PKG_VERSION"), - " at your service." - )) - .to_string(), - - ServerProtocol::Jmap => self - .value_or_else(("server.listener", id, "url"), "server.url") - .failed(&format!("No 'url' directive found for listener {id:?}")) - .to_string(), - ServerProtocol::Imap | ServerProtocol::Http | ServerProtocol::ManageSieve => self - .value_or_else(("server.listener", id, "url"), "server.url") - .unwrap_or_default() - .to_string(), - }, - max_connections: self - .property_or_else( - ("server.listener", id, "max-connections"), - "server.max-connections", - )? - .unwrap_or(8192), - protocol, - listeners, - acceptor, - tls_implicit, - proxy_networks, - }) - } -} - -impl ParseValue for ServerProtocol { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - if value.eq_ignore_ascii_case("smtp") { - Ok(Self::Smtp) - } else if value.eq_ignore_ascii_case("lmtp") { - Ok(Self::Lmtp) - } else if value.eq_ignore_ascii_case("jmap") { - Ok(Self::Jmap) - } else if value.eq_ignore_ascii_case("imap") { - Ok(Self::Imap) - } else if value.eq_ignore_ascii_case("http") | value.eq_ignore_ascii_case("https") { - Ok(Self::Http) - } else if value.eq_ignore_ascii_case("managesieve") { - Ok(Self::ManageSieve) - } else { - Err(format!( - "Invalid server protocol type {:?} for property {:?}.", - value, - key.as_key() - )) - } - } -} - -impl ParseValue for SocketAddr { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - value.parse().map_err(|_| { - format!( - "Invalid socket address {:?} for property {:?}.", - value, - key.as_key() - ) - }) - } -} - -impl ParseValue for SupportedCipherSuite { - fn parse_value(key: impl AsKey, value: &str) -> super::Result { - Ok(match value { - // TLS1.3 suites - "TLS13_AES_256_GCM_SHA384" => TLS13_AES_256_GCM_SHA384, - "TLS13_AES_128_GCM_SHA256" => TLS13_AES_128_GCM_SHA256, - "TLS13_CHACHA20_POLY1305_SHA256" => TLS13_CHACHA20_POLY1305_SHA256, - // TLS1.2 suites - "TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384" => TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256" => TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" => { - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 - } - "TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384" => TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256" => TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - "TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256" => { - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 - } - cipher => { - return Err(format!( - "Unsupported TLS cipher suite {:?} found in key {:?}", - cipher, - key.as_key() - )) - } - }) - } -} diff --git a/crates/utils/src/config/mod.rs b/crates/utils/src/config/mod.rs index 702811d7..3cd303f2 100644 --- a/crates/utils/src/config/mod.rs +++ b/crates/utils/src/config/mod.rs @@ -22,26 +22,15 @@ */ pub mod cron; -pub mod if_block; pub mod ipmask; -pub mod listener; pub mod parser; -pub mod tls; pub mod utils; -use std::{collections::BTreeMap, fmt::Display, net::SocketAddr, sync::Arc, time::Duration}; +use std::{collections::BTreeMap, time::Duration}; use ahash::AHashMap; -use tokio::net::TcpSocket; -use crate::{ - acme::AcmeManager, - failed, - listener::{tls::Certificate, TcpAcceptor}, - UnwrapFailure, -}; - -use self::ipmask::IpAddrMask; +use crate::{failed, UnwrapFailure}; #[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct Config { @@ -63,69 +52,12 @@ pub struct ConfigKey { pub value: String, } -#[derive(Debug, Default)] -pub struct Server { - pub id: String, - pub internal_id: u16, - pub hostname: String, - pub data: String, - pub protocol: ServerProtocol, - pub listeners: Vec, - pub proxy_networks: Vec, - pub acceptor: TcpAcceptor, - pub tls_implicit: bool, - pub max_connections: u64, -} - -#[derive(Default)] -pub struct Servers { - pub inner: Vec, - pub certificates: Vec>, - pub acme_managers: Vec>, -} - -#[derive(Debug)] -pub struct Listener { - pub socket: TcpSocket, - pub addr: SocketAddr, - pub backlog: Option, - - // TCP options - pub ttl: Option, - pub linger: Option, - pub nodelay: bool, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy, Default)] -pub enum ServerProtocol { - #[default] - Smtp, - Lmtp, - Jmap, - Imap, - Http, - ManageSieve, -} - #[derive(Debug, Default, PartialEq, Eq, Clone)] pub struct Rate { pub requests: u64, pub period: Duration, } -impl Display for ServerProtocol { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ServerProtocol::Smtp => write!(f, "smtp"), - ServerProtocol::Lmtp => write!(f, "lmtp"), - ServerProtocol::Jmap => write!(f, "jmap"), - ServerProtocol::Imap => write!(f, "imap"), - ServerProtocol::Http => write!(f, "http"), - ServerProtocol::ManageSieve => write!(f, "managesieve"), - } - } -} - pub type Result = std::result::Result; impl Config { @@ -166,23 +98,25 @@ impl Config { } pub async fn resolve_macros(&mut self) { + for macro_class in ["env", "file", "cfg"] { + self.resolve_macro_type(macro_class).await; + } + } + + async fn resolve_macro_type(&mut self, class: &str) { + let macro_start = format!("%{{{class}:"); let mut replacements = AHashMap::new(); 'outer: for (key, value) in &self.keys { - if value.contains("%{") && value.contains("}%") { + if value.contains(¯o_start) && value.contains("}%") { let mut result = String::with_capacity(value.len()); let mut snippet: &str = value.as_str(); loop { - if let Some((suffix, macro_name)) = snippet.split_once("%{") { + if let Some((suffix, macro_name)) = snippet.split_once(¯o_start) { if !suffix.is_empty() { result.push_str(suffix); } - if let Some((class, location, rest)) = - macro_name.split_once("}%").and_then(|(name, rest)| { - name.split_once(':') - .map(|(class, location)| (class, location, rest)) - }) - { + if let Some((location, rest)) = macro_name.split_once("}%") { match class { "cfg" => { if let Some(value) = replacements @@ -238,9 +172,8 @@ impl Config { } } } - "http" | "https" => {} _ => { - continue 'outer; + unreachable!() } }; @@ -266,4 +199,43 @@ impl Config { pub fn update(&mut self, settings: Vec<(String, String)>) { self.keys.extend(settings); } + + pub fn log_errors(&self, use_stderr: bool) { + for (key, err) in &self.errors { + let message = match err { + ConfigError::Parse(err) => { + format!("Failed to parse setting {key:?}: {err}") + } + ConfigError::Build(err) => { + format!("Build error for key {key:?}: {err}") + } + ConfigError::Macro(err) => { + format!("Macro expansion error for setting {key:?}: {err}") + } + }; + if !use_stderr { + tracing::error!("{}", message); + } else { + eprintln!("ERROR: {message}"); + } + } + } + + pub fn log_warnings(&self, use_stderr: bool) { + for (key, value) in &self.missing { + let message = match value { + Some(replaced) => { + format!("WARNING: Missing setting {key:?}, applied default {replaced:?}") + } + None => { + format!("WARNING: Missing setting {key:?}") + } + }; + if !use_stderr { + tracing::debug!("{}", message); + } else { + eprintln!("{}", message); + } + } + } } diff --git a/crates/utils/src/config/tls.rs b/crates/utils/src/config/tls.rs deleted file mode 100644 index 2468bb04..00000000 --- a/crates/utils/src/config/tls.rs +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{io::Cursor, path::PathBuf, sync::Arc, time::Duration}; - -use ahash::AHashMap; -use arc_swap::ArcSwap; -use rcgen::generate_simple_self_signed; -use rustls::{ - crypto::ring::sign::any_supported_type, - sign::CertifiedKey, - version::{TLS12, TLS13}, - SupportedProtocolVersion, -}; -use rustls_pemfile::{certs, read_one, Item}; -use rustls_pki_types::PrivateKeyDer; - -use crate::{ - acme::{directory::LETS_ENCRYPT_PRODUCTION_DIRECTORY, AcmeManager}, - listener::tls::Certificate, -}; - -use super::Config; - -pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; -pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; - -impl Config { - pub fn parse_certificates(&self) -> super::Result>> { - let mut certs = AHashMap::new(); - for cert_id in self.sub_keys("certificate", ".cert") { - let key_cert = ("certificate", cert_id, "cert"); - let key_pk = ("certificate", cert_id, "private-key"); - - let mut cert = Certificate { - cert: ArcSwap::from(Arc::new(build_certified_key( - self.value_require(key_cert)?.as_bytes().to_vec(), - self.value_require(key_pk)?.as_bytes().to_vec(), - &format!("certificate.{cert_id}"), - )?)), - path: Vec::with_capacity(2), - }; - - for key in [key_cert, key_pk] { - if let Some(path) = self.value(key).and_then(|v| v.strip_prefix("file://")) { - cert.path.push(PathBuf::from(path)); - } - } - - certs.insert(cert_id.to_string(), Arc::new(cert)); - } - - Ok(certs) - } - - pub fn parse_acmes(&self) -> super::Result>> { - let mut acmes = AHashMap::new(); - for acme_id in self.sub_keys("acme", ".cache") { - let directory = self - .value(("acme", acme_id, "directory")) - .unwrap_or(LETS_ENCRYPT_PRODUCTION_DIRECTORY) - .trim() - .to_string(); - let contact = self - .values(("acme", acme_id, "contact")) - .filter_map(|(_, v)| { - let v = v.trim().to_string(); - if !v.is_empty() { - Some(v) - } else { - None - } - }) - .collect::>(); - let cache = PathBuf::from(self.value_require(("acme", acme_id, "cache"))?); - let renew_before: Duration = - self.property_or_default(("acme", acme_id, "renew-before"), "30d")?; - - if directory.is_empty() { - return Err(format!("Missing directory for acme.{acme_id}.")); - } - - if contact.is_empty() { - return Err(format!("Missing contact for acme.{acme_id}.")); - } - - // Find which domains are covered by this ACME manager - let mut domains = Vec::new(); - for id in self.sub_keys("server.listener", ".protocol") { - match ( - self.value_or_else(("server.listener", id, "tls.acme"), "server.tls.acme"), - self.value_or_else(("server.listener", id, "hostname"), "server.hostname"), - ) { - (Some(listener_acme), Some(hostname)) if listener_acme == acme_id => { - let hostname = hostname.trim().to_lowercase(); - - if !domains.contains(&hostname) { - domains.push(hostname); - } - } - _ => (), - } - } - - acmes.insert( - acme_id.to_string(), - Arc::new(AcmeManager::new( - directory, - domains, - contact, - renew_before, - cache, - )?), - ); - } - - Ok(acmes) - } -} - -pub(crate) fn build_certified_key( - cert: Vec, - pk: Vec, - id: &str, -) -> super::Result { - let cert = certs(&mut Cursor::new(cert)) - .collect::, _>>() - .map_err(|err| format!("Failed to read certificates in {id:?}: {err}"))?; - if cert.is_empty() { - return Err(format!("No certificates found in {id:?}.")); - } - let pk = match read_one(&mut Cursor::new(pk)) - .map_err(|err| format!("Failed to read private keys in {id:?}.: {err}",))? - .into_iter() - .next() - { - Some(Item::Pkcs8Key(key)) => PrivateKeyDer::Pkcs8(key), - Some(Item::Pkcs1Key(key)) => PrivateKeyDer::Pkcs1(key), - Some(Item::Sec1Key(key)) => PrivateKeyDer::Sec1(key), - Some(_) => return Err(format!("Unsupported private keys found in {id:?}.",)), - None => return Err(format!("No private keys found in {id:?}.",)), - }; - - Ok(CertifiedKey { - cert, - key: any_supported_type(&pk) - .map_err(|err| format!("Failed to sign certificate for {id:?}: {err}",))?, - ocsp: None, - }) -} - -pub(crate) fn build_self_signed_cert(domains: &[String]) -> super::Result { - let cert = generate_simple_self_signed(domains).map_err(|err| { - format!( - "Failed to generate self-signed certificate for {domains:?}: {err}", - domains = domains - ) - })?; - build_certified_key( - cert.serialize_pem().unwrap().into_bytes(), - cert.serialize_private_key_pem().into_bytes(), - "self-signed", - ) -} diff --git a/crates/utils/src/config/utils.rs b/crates/utils/src/config/utils.rs index e279262a..65d77700 100644 --- a/crates/utils/src/config/utils.rs +++ b/crates/utils/src/config/utils.rs @@ -34,8 +34,6 @@ use mail_auth::{ }; use smtp_proto::MtPriority; -use crate::expr::{Constant, Variable}; - use super::{Config, ConfigError, Rate}; impl Config { @@ -351,11 +349,6 @@ pub trait ParseValue: Sized { fn parse_value(key: impl AsKey, value: &str) -> super::Result; } -pub trait ConstantValue: - ParseValue + for<'x> TryFrom> + Into + Sized -{ -} - pub trait ParseKey { fn parse_key(&self, key: impl AsKey) -> super::Result; } @@ -551,35 +544,6 @@ impl ParseValue for MtPriority { } } -impl<'x> TryFrom> for MtPriority { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Integer(value) => match value { - 2 => Ok(MtPriority::Mixer), - 3 => Ok(MtPriority::Stanag4406), - 4 => Ok(MtPriority::Nsep), - _ => Err(()), - }, - Variable::String(value) => MtPriority::parse_value("", &value).map_err(|_| ()), - _ => Err(()), - } - } -} - -impl From for Constant { - fn from(value: MtPriority) -> Self { - Constant::Integer(match value { - MtPriority::Mixer => 2, - MtPriority::Stanag4406 => 3, - MtPriority::Nsep => 4, - }) - } -} - -impl ConstantValue for MtPriority {} - impl ParseValue for Canonicalization { fn parse_value(key: impl AsKey, value: &str) -> super::Result { match value { @@ -613,37 +577,6 @@ impl ParseValue for IpLookupStrategy { } } -impl<'x> TryFrom> for IpLookupStrategy { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Integer(value) => match value { - 2 => Ok(IpLookupStrategy::Ipv4Only), - 3 => Ok(IpLookupStrategy::Ipv6Only), - 4 => Ok(IpLookupStrategy::Ipv6thenIpv4), - 5 => Ok(IpLookupStrategy::Ipv4thenIpv6), - _ => Err(()), - }, - Variable::String(value) => IpLookupStrategy::parse_value("", &value).map_err(|_| ()), - _ => Err(()), - } - } -} - -impl From for Constant { - fn from(value: IpLookupStrategy) -> Self { - Constant::Integer(match value { - IpLookupStrategy::Ipv4Only => 2, - IpLookupStrategy::Ipv6Only => 3, - IpLookupStrategy::Ipv6thenIpv4 => 4, - IpLookupStrategy::Ipv4thenIpv6 => 5, - }) - } -} - -impl ConstantValue for IpLookupStrategy {} - impl ParseValue for Algorithm { fn parse_value(key: impl AsKey, value: &str) -> super::Result { match value { @@ -721,124 +654,6 @@ impl ParseValue for Duration { } } -impl ConstantValue for Duration {} - -impl<'x> TryFrom> for Duration { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Integer(value) if value > 0 => Ok(Duration::from_millis(value as u64)), - Variable::Float(value) if value > 0.0 => Ok(Duration::from_millis(value as u64)), - Variable::String(value) if !value.is_empty() => { - Duration::parse_value("", &value).map_err(|_| ()) - } - _ => Err(()), - } - } -} - -impl From for Constant { - fn from(value: Duration) -> Self { - Constant::Integer(value.as_millis() as i64) - } -} - -impl<'x> TryFrom> for Rate { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::Array(items) if items.len() == 2 => { - let requests = items[0].to_integer().ok_or(())?; - let period = items[1].to_integer().ok_or(())?; - - if requests > 0 && period > 0 { - Ok(Rate { - requests: requests as u64, - period: Duration::from_millis(period as u64), - }) - } else { - Err(()) - } - } - _ => Err(()), - } - } -} - -impl<'x> TryFrom> for Ipv4Addr { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::String(value) => value.parse().map_err(|_| ()), - _ => Err(()), - } - } -} - -impl<'x> TryFrom> for Ipv6Addr { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::String(value) => value.parse().map_err(|_| ()), - _ => Err(()), - } - } -} - -impl<'x> TryFrom> for IpAddr { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - match value { - Variable::String(value) => value.parse().map_err(|_| ()), - _ => Err(()), - } - } -} - -impl<'x, T: TryFrom>> TryFrom> for Vec -where - Result, ()>: FromIterator>>::Error>>, -{ - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - value - .into_array() - .into_iter() - .map(|v| T::try_from(v)) - .collect() - } -} - -pub struct NoConstants; - -impl<'x> TryFrom> for NoConstants { - type Error = (); - - fn try_from(_: Variable<'x>) -> Result { - Err(()) - } -} - -impl From for Constant { - fn from(_: NoConstants) -> Self { - Constant::Integer(0) - } -} - -impl ParseValue for NoConstants { - fn parse_value(_: impl AsKey, _: &str) -> super::Result { - Err("".to_string()) - } -} - -impl ConstantValue for NoConstants {} - impl ParseValue for Rate { fn parse_value(key: impl AsKey, value: &str) -> super::Result { if let Some((requests, period)) = value.split_once('/') { diff --git a/crates/utils/src/expr/eval.rs b/crates/utils/src/expr/eval.rs deleted file mode 100644 index 453212f9..00000000 --- a/crates/utils/src/expr/eval.rs +++ /dev/null @@ -1,569 +0,0 @@ -/* - * Copyright (c) 2020-2023, Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{borrow::Cow, cmp::Ordering, fmt::Display}; - -use super::{ - functions::FUNCTIONS, BinaryOperator, Constant, Expression, ExpressionItem, UnaryOperator, - Variable, -}; - -impl Expression { - pub async fn eval<'x, 'y, V, F, R>( - &'x self, - var: V, - mut fnc: F, - captures: &'y mut Vec, - ) -> Variable<'x> - where - V: Fn(u32) -> Variable<'x>, - F: FnMut(u32, Vec>) -> R, - R: std::future::Future> + Send, - { - let mut stack = Vec::new(); - let mut exprs = self.items.iter(); - - while let Some(expr) = exprs.next() { - match expr { - ExpressionItem::Variable(v) => { - stack.push(var(*v)); - } - ExpressionItem::Constant(val) => { - stack.push(Variable::from(val)); - } - ExpressionItem::Capture(v) => { - stack.push(Variable::String(Cow::Owned( - captures - .get(*v as usize) - .map(|v| v.as_str()) - .unwrap_or_default() - .to_string(), - ))); - } - ExpressionItem::UnaryOperator(op) => { - let value = stack.pop().unwrap_or_default(); - stack.push(match op { - UnaryOperator::Not => value.op_not(), - UnaryOperator::Minus => value.op_minus(), - }); - } - ExpressionItem::BinaryOperator(op) => { - let right = stack.pop().unwrap_or_default(); - let left = stack.pop().unwrap_or_default(); - stack.push(match op { - BinaryOperator::Add => left.op_add(right), - BinaryOperator::Subtract => left.op_subtract(right), - BinaryOperator::Multiply => left.op_multiply(right), - BinaryOperator::Divide => left.op_divide(right), - BinaryOperator::And => left.op_and(right), - BinaryOperator::Or => left.op_or(right), - BinaryOperator::Xor => left.op_xor(right), - BinaryOperator::Eq => left.op_eq(right), - BinaryOperator::Ne => left.op_ne(right), - BinaryOperator::Lt => left.op_lt(right), - BinaryOperator::Le => left.op_le(right), - BinaryOperator::Gt => left.op_gt(right), - BinaryOperator::Ge => left.op_ge(right), - }); - } - ExpressionItem::Function { id, num_args } => { - let num_args = *num_args as usize; - - let mut arguments = Variable::array(num_args); - for arg_num in 0..num_args { - arguments[num_args - arg_num - 1] = stack.pop().unwrap_or_default(); - } - - let result = if let Some((_, fnc, _)) = FUNCTIONS.get(*id as usize) { - (fnc)(arguments) - } else { - (fnc)(*id - FUNCTIONS.len() as u32, arguments).await - }; - - stack.push(result); - } - ExpressionItem::JmpIf { val, pos } => { - if stack.last().map_or(false, |v| v.to_bool()) == *val { - for _ in 0..*pos { - exprs.next(); - } - } - } - ExpressionItem::ArrayAccess => { - let index = stack - .pop() - .unwrap_or_default() - .to_usize() - .unwrap_or_default(); - let array = stack.pop().unwrap_or_default().into_array(); - stack.push(array.into_iter().nth(index).unwrap_or_default()); - } - ExpressionItem::ArrayBuild(num_items) => { - let num_items = *num_items as usize; - let mut items = Variable::array(num_items); - for arg_num in 0..num_items { - items[num_items - arg_num - 1] = stack.pop().unwrap_or_default(); - } - stack.push(Variable::Array(items)); - } - ExpressionItem::Regex(regex) => { - captures.clear(); - let value = stack.pop().unwrap_or_default().into_string(); - - if let Some(captures_) = regex.captures(value.as_ref()) { - for capture in captures_.iter() { - captures.push(capture.map_or("", |m| m.as_str()).to_string()); - } - } - - stack.push(Variable::Integer(!captures.is_empty() as i64)); - } - } - } - - stack.pop().unwrap_or_default() - } - - pub fn is_empty(&self) -> bool { - self.items.is_empty() - } - - pub fn items(&self) -> &[ExpressionItem] { - &self.items - } -} - -impl<'x> Variable<'x> { - pub fn op_add(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_add(b)), - (Variable::Float(a), Variable::Float(b)) => Variable::Float(a + b), - (Variable::Integer(i), Variable::Float(f)) - | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 + f), - (Variable::Array(a), Variable::Array(b)) => { - Variable::Array(a.into_iter().chain(b).collect::>()) - } - (Variable::Array(a), b) => { - Variable::Array(a.into_iter().chain([b]).collect::>()) - } - (a, Variable::Array(b)) => { - Variable::Array([a].into_iter().chain(b).collect::>()) - } - (Variable::String(a), b) => { - if !a.is_empty() { - Variable::String(format!("{}{}", a, b).into()) - } else { - b - } - } - (a, Variable::String(b)) => { - if !b.is_empty() { - Variable::String(format!("{}{}", a, b).into()) - } else { - a - } - } - } - } - - pub fn op_subtract(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_sub(b)), - (Variable::Float(a), Variable::Float(b)) => Variable::Float(a - b), - (Variable::Integer(a), Variable::Float(b)) => Variable::Float(a as f64 - b), - (Variable::Float(a), Variable::Integer(b)) => Variable::Float(a - b as f64), - (Variable::Array(a), b) | (b, Variable::Array(a)) => { - Variable::Array(a.into_iter().filter(|v| v != &b).collect::>()) - } - (a, b) => a.parse_number().op_subtract(b.parse_number()), - } - } - - pub fn op_multiply(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => Variable::Integer(a.saturating_mul(b)), - (Variable::Float(a), Variable::Float(b)) => Variable::Float(a * b), - (Variable::Integer(i), Variable::Float(f)) - | (Variable::Float(f), Variable::Integer(i)) => Variable::Float(i as f64 * f), - (a, b) => a.parse_number().op_multiply(b.parse_number()), - } - } - - pub fn op_divide(self, other: Variable<'x>) -> Variable<'x> { - match (self, other) { - (Variable::Integer(a), Variable::Integer(b)) => { - Variable::Float(if b != 0 { a as f64 / b as f64 } else { 0.0 }) - } - (Variable::Float(a), Variable::Float(b)) => { - Variable::Float(if b != 0.0 { a / b } else { 0.0 }) - } - (Variable::Integer(a), Variable::Float(b)) => { - Variable::Float(if b != 0.0 { a as f64 / b } else { 0.0 }) - } - (Variable::Float(a), Variable::Integer(b)) => { - Variable::Float(if b != 0 { a / b as f64 } else { 0.0 }) - } - (a, b) => a.parse_number().op_divide(b.parse_number()), - } - } - - pub fn op_and(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self.to_bool() & other.to_bool())) - } - - pub fn op_or(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self.to_bool() | other.to_bool())) - } - - pub fn op_xor(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self.to_bool() ^ other.to_bool())) - } - - pub fn op_eq(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self == other)) - } - - pub fn op_ne(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self != other)) - } - - pub fn op_lt(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self < other)) - } - - pub fn op_le(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self <= other)) - } - - pub fn op_gt(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self > other)) - } - - pub fn op_ge(self, other: Variable) -> Variable { - Variable::Integer(i64::from(self >= other)) - } - - pub fn op_not(self) -> Variable<'static> { - Variable::Integer(i64::from(!self.to_bool())) - } - - pub fn op_minus(self) -> Variable<'static> { - match self { - Variable::Integer(n) => Variable::Integer(-n), - Variable::Float(n) => Variable::Float(-n), - _ => self.parse_number().op_minus(), - } - } - - pub fn parse_number(&self) -> Variable<'static> { - match self { - Variable::String(s) if !s.is_empty() => { - if let Ok(n) = s.parse::() { - Variable::Integer(n) - } else if let Ok(n) = s.parse::() { - Variable::Float(n) - } else { - Variable::Integer(0) - } - } - Variable::Integer(n) => Variable::Integer(*n), - Variable::Float(n) => Variable::Float(*n), - Variable::Array(l) => Variable::Integer(l.is_empty() as i64), - _ => Variable::Integer(0), - } - } - - #[inline(always)] - fn array(num_items: usize) -> Vec> { - let mut items = Vec::with_capacity(num_items); - for _ in 0..num_items { - items.push(Variable::Integer(0)); - } - items - } - - pub fn to_ref<'y: 'x>(&'y self) -> Variable<'x> { - match self { - Variable::String(s) => Variable::String(Cow::Borrowed(s.as_ref())), - Variable::Integer(n) => Variable::Integer(*n), - Variable::Float(n) => Variable::Float(*n), - Variable::Array(l) => Variable::Array(l.iter().map(|v| v.to_ref()).collect::>()), - } - } - - pub fn to_bool(&self) -> bool { - match self { - Variable::Float(f) => *f != 0.0, - Variable::Integer(n) => *n != 0, - Variable::String(s) => !s.is_empty(), - Variable::Array(a) => !a.is_empty(), - } - } - - pub fn to_string(&self) -> Cow<'_, str> { - match self { - Variable::String(s) => Cow::Borrowed(s.as_ref()), - Variable::Integer(n) => Cow::Owned(n.to_string()), - Variable::Float(n) => Cow::Owned(n.to_string()), - Variable::Array(l) => { - let mut result = String::with_capacity(self.len() * 10); - for item in l { - if !result.is_empty() { - result.push_str("\r\n"); - } - match item { - Variable::String(v) => result.push_str(v), - Variable::Integer(v) => result.push_str(&v.to_string()), - Variable::Float(v) => result.push_str(&v.to_string()), - Variable::Array(_) => {} - } - } - Cow::Owned(result) - } - } - } - - pub fn into_string(self) -> Cow<'x, str> { - match self { - Variable::String(s) => s, - Variable::Integer(n) => Cow::Owned(n.to_string()), - Variable::Float(n) => Cow::Owned(n.to_string()), - Variable::Array(l) => { - let mut result = String::with_capacity(l.len() * 10); - for item in l { - if !result.is_empty() { - result.push_str("\r\n"); - } - match item { - Variable::String(v) => result.push_str(v.as_ref()), - Variable::Integer(v) => result.push_str(&v.to_string()), - Variable::Float(v) => result.push_str(&v.to_string()), - Variable::Array(_) => {} - } - } - Cow::Owned(result) - } - } - } - - pub fn to_integer(&self) -> Option { - match self { - Variable::Integer(n) => Some(*n), - Variable::Float(n) => Some(*n as i64), - Variable::String(s) if !s.is_empty() => s.parse::().ok(), - _ => None, - } - } - - pub fn to_usize(&self) -> Option { - match self { - Variable::Integer(n) => Some(*n as usize), - Variable::Float(n) => Some(*n as usize), - Variable::String(s) if !s.is_empty() => s.parse::().ok(), - _ => None, - } - } - - pub fn len(&self) -> usize { - match self { - Variable::String(s) => s.len(), - Variable::Integer(_) | Variable::Float(_) => 2, - Variable::Array(l) => l.iter().map(|v| v.len() + 2).sum(), - } - } - - pub fn is_empty(&self) -> bool { - match self { - Variable::String(s) => s.is_empty(), - _ => false, - } - } - - pub fn as_array(&self) -> Option<&[Variable]> { - match self { - Variable::Array(l) => Some(l), - _ => None, - } - } - - pub fn into_array(self) -> Vec> { - match self { - Variable::Array(l) => l, - v if !v.is_empty() => vec![v], - _ => vec![], - } - } - - pub fn to_array(&self) -> Vec> { - match self { - Variable::Array(l) => l.iter().map(|v| v.to_ref()).collect::>(), - v if !v.is_empty() => vec![v.to_ref()], - _ => vec![], - } - } - - pub fn into_owned(self) -> Variable<'static> { - match self { - Variable::String(s) => Variable::String(Cow::Owned(s.into_owned())), - Variable::Integer(n) => Variable::Integer(n), - Variable::Float(n) => Variable::Float(n), - Variable::Array(l) => Variable::Array(l.into_iter().map(|v| v.into_owned()).collect()), - } - } -} - -impl PartialEq for Variable<'_> { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Integer(a), Self::Integer(b)) => a == b, - (Self::Float(a), Self::Float(b)) => a == b, - (Self::Integer(a), Self::Float(b)) | (Self::Float(b), Self::Integer(a)) => { - *a as f64 == *b - } - (Self::String(a), Self::String(b)) => a == b, - (Self::String(_), Self::Integer(_) | Self::Float(_)) => &self.parse_number() == other, - (Self::Integer(_) | Self::Float(_), Self::String(_)) => self == &other.parse_number(), - (Self::Array(a), Self::Array(b)) => a == b, - _ => false, - } - } -} - -impl Eq for Variable<'_> {} - -#[allow(clippy::non_canonical_partial_ord_impl)] -impl PartialOrd for Variable<'_> { - fn partial_cmp(&self, other: &Self) -> Option { - match (self, other) { - (Self::Integer(a), Self::Integer(b)) => a.partial_cmp(b), - (Self::Float(a), Self::Float(b)) => a.partial_cmp(b), - (Self::Integer(a), Self::Float(b)) => (*a as f64).partial_cmp(b), - (Self::Float(a), Self::Integer(b)) => a.partial_cmp(&(*b as f64)), - (Self::String(a), Self::String(b)) => a.partial_cmp(b), - (Self::String(_), Self::Integer(_) | Self::Float(_)) => { - self.parse_number().partial_cmp(other) - } - (Self::Integer(_) | Self::Float(_), Self::String(_)) => { - self.partial_cmp(&other.parse_number()) - } - (Self::Array(a), Self::Array(b)) => a.partial_cmp(b), - (Self::Array(_) | Self::String(_), _) => Ordering::Greater.into(), - (_, Self::Array(_)) => Ordering::Less.into(), - } - } -} - -impl Ord for Variable<'_> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.partial_cmp(other).unwrap_or(Ordering::Greater) - } -} - -impl Display for Variable<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Variable::String(v) => v.fmt(f), - Variable::Integer(v) => v.fmt(f), - Variable::Float(v) => v.fmt(f), - Variable::Array(v) => { - for (i, v) in v.iter().enumerate() { - if i > 0 { - f.write_str("\n")?; - } - v.fmt(f)?; - } - Ok(()) - } - } - } -} - -trait IntoBool { - fn into_bool(self) -> bool; -} - -impl IntoBool for f64 { - #[inline(always)] - fn into_bool(self) -> bool { - self != 0.0 - } -} - -impl IntoBool for i64 { - #[inline(always)] - fn into_bool(self) -> bool { - self != 0 - } -} - -impl<'x> From<&'x Constant> for Variable<'x> { - fn from(value: &'x Constant) -> Self { - match value { - Constant::Integer(i) => Variable::Integer(*i), - Constant::Float(f) => Variable::Float(*f), - Constant::String(s) => Variable::String(s.as_str().into()), - } - } -} - -impl<'x> TryFrom> for String { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - if let Variable::String(s) = value { - Ok(s.into_owned()) - } else { - Err(()) - } - } -} - -impl<'x> From> for bool { - fn from(val: Variable<'x>) -> Self { - val.to_bool() - } -} - -impl<'x> TryFrom> for i64 { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - value.to_integer().ok_or(()) - } -} - -impl<'x> TryFrom> for u64 { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - value.to_integer().map(|v| v as u64).ok_or(()) - } -} - -impl<'x> TryFrom> for usize { - type Error = (); - - fn try_from(value: Variable<'x>) -> Result { - value.to_usize().ok_or(()) - } -} diff --git a/crates/utils/src/expr/functions/array.rs b/crates/utils/src/expr/functions/array.rs deleted file mode 100644 index 12868067..00000000 --- a/crates/utils/src/expr/functions/array.rs +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use crate::expr::Variable; - -pub(crate) fn fn_count(v: Vec) -> Variable { - match &v[0] { - Variable::Array(a) => a.len(), - v => { - if !v.is_empty() { - 1 - } else { - 0 - } - } - } - .into() -} - -pub(crate) fn fn_sort(mut v: Vec) -> Variable { - let is_asc = v[1].to_bool(); - let mut arr = v.remove(0).into_array(); - if is_asc { - arr.sort_unstable_by(|a, b| b.cmp(a)); - } else { - arr.sort_unstable(); - } - arr.into() -} - -pub(crate) fn fn_dedup(mut v: Vec) -> Variable { - let arr = v.remove(0).into_array(); - let mut result = Vec::with_capacity(arr.len()); - - for item in arr { - if !result.contains(&item) { - result.push(item); - } - } - - result.into() -} - -pub(crate) fn fn_is_intersect(v: Vec) -> Variable { - match (&v[0], &v[1]) { - (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), - (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), - _ => false, - } - .into() -} - -pub(crate) fn fn_winnow(mut v: Vec) -> Variable { - match v.remove(0) { - Variable::Array(a) => a - .into_iter() - .filter(|i| !i.is_empty()) - .collect::>() - .into(), - v => v, - } -} diff --git a/crates/utils/src/expr/functions/email.rs b/crates/utils/src/expr/functions/email.rs deleted file mode 100644 index 42a6d318..00000000 --- a/crates/utils/src/expr/functions/email.rs +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::borrow::Cow; - -use crate::expr::Variable; - -pub(crate) fn fn_is_email(v: Vec) -> Variable { - let mut last_ch = 0; - let mut in_quote = false; - let mut at_count = 0; - let mut dot_count = 0; - let mut lp_len = 0; - let mut value = 0; - - for ch in v[0].to_string().bytes() { - match ch { - b'0'..=b'9' - | b'a'..=b'z' - | b'A'..=b'Z' - | b'!' - | b'#' - | b'$' - | b'%' - | b'&' - | b'\'' - | b'*' - | b'+' - | b'-' - | b'/' - | b'=' - | b'?' - | b'^' - | b'_' - | b'`' - | b'{' - | b'|' - | b'}' - | b'~' - | 0x7f..=u8::MAX => { - value += 1; - } - b'.' if !in_quote => { - if last_ch != b'.' && last_ch != b'@' && value != 0 { - value += 1; - if at_count == 1 { - dot_count += 1; - } - } else { - return false.into(); - } - } - b'@' if !in_quote => { - at_count += 1; - lp_len = value; - value = 0; - } - b'>' | b':' | b',' | b' ' if in_quote => { - value += 1; - } - b'\"' if !in_quote || last_ch != b'\\' => { - in_quote = !in_quote; - } - b'\\' if in_quote && last_ch != b'\\' => (), - _ => { - if !in_quote { - return false.into(); - } - } - } - - last_ch = ch; - } - - (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() -} - -pub(crate) fn fn_email_part(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap(); - let part = v.next().unwrap().into_string(); - - value.transform(|s| match s { - Cow::Borrowed(s) => s - .rsplit_once('@') - .map(|(u, d)| match part.as_ref() { - "local" => Variable::from(u.trim()), - "domain" => Variable::from(d.trim()), - _ => Variable::default(), - }) - .unwrap_or_default(), - Cow::Owned(s) => s - .rsplit_once('@') - .map(|(u, d)| match part.as_ref() { - "local" => Variable::from(u.trim().to_string()), - "domain" => Variable::from(d.trim().to_string()), - _ => Variable::default(), - }) - .unwrap_or_default(), - }) -} diff --git a/crates/utils/src/expr/functions/misc.rs b/crates/utils/src/expr/functions/misc.rs deleted file mode 100644 index 6947abf4..00000000 --- a/crates/utils/src/expr/functions/misc.rs +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::net::IpAddr; - -use mail_auth::common::resolver::ToReverseName; - -use crate::expr::Variable; - -pub(crate) fn fn_is_empty(v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.is_empty(), - Variable::Integer(_) | Variable::Float(_) => false, - Variable::Array(a) => a.is_empty(), - } - .into() -} - -pub(crate) fn fn_is_number(v: Vec) -> Variable { - matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into() -} - -pub(crate) fn fn_is_ip_addr(v: Vec) -> Variable { - v[0].to_string().parse::().is_ok().into() -} - -pub(crate) fn fn_is_ipv4_addr(v: Vec) -> Variable { - v[0].to_string() - .parse::() - .map_or(false, |ip| matches!(ip, IpAddr::V4(_))) - .into() -} - -pub(crate) fn fn_is_ipv6_addr(v: Vec) -> Variable { - v[0].to_string() - .parse::() - .map_or(false, |ip| matches!(ip, IpAddr::V6(_))) - .into() -} - -pub(crate) fn fn_ip_reverse_name(v: Vec) -> Variable { - v[0].to_string() - .parse::() - .map(|ip| ip.to_reverse_name()) - .unwrap_or_default() - .into() -} diff --git a/crates/utils/src/expr/functions/mod.rs b/crates/utils/src/expr/functions/mod.rs deleted file mode 100644 index 9effffac..00000000 --- a/crates/utils/src/expr/functions/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::borrow::Cow; - -use super::Variable; - -pub mod array; -pub mod email; -pub mod misc; -pub mod text; - -impl<'x> Variable<'x> { - fn transform(self, f: impl Fn(Cow<'x, str>) -> Variable<'x>) -> Variable<'x> { - match self { - Variable::String(s) => f(s), - Variable::Array(list) => Variable::Array( - list.into_iter() - .map(|v| match v { - Variable::String(s) => f(s), - v => f(v.into_string()), - }) - .collect::>(), - ), - v => f(v.into_string()), - } - } -} - -#[allow(clippy::type_complexity)] -pub(crate) const FUNCTIONS: &[(&str, fn(Vec) -> Variable, u32)] = &[ - ("count", array::fn_count, 1), - ("sort", array::fn_sort, 2), - ("dedup", array::fn_dedup, 1), - ("winnow", array::fn_winnow, 1), - ("is_intersect", array::fn_is_intersect, 2), - ("is_email", email::fn_is_email, 1), - ("email_part", email::fn_email_part, 2), - ("is_empty", misc::fn_is_empty, 1), - ("is_number", misc::fn_is_number, 1), - ("is_ip_addr", misc::fn_is_ip_addr, 1), - ("is_ipv4_addr", misc::fn_is_ipv4_addr, 1), - ("is_ipv6_addr", misc::fn_is_ipv6_addr, 1), - ("ip_reverse_name", misc::fn_ip_reverse_name, 1), - ("trim", text::fn_trim, 1), - ("trim_end", text::fn_trim_end, 1), - ("trim_start", text::fn_trim_start, 1), - ("len", text::fn_len, 1), - ("to_lowercase", text::fn_to_lowercase, 1), - ("to_uppercase", text::fn_to_uppercase, 1), - ("is_uppercase", text::fn_is_uppercase, 1), - ("is_lowercase", text::fn_is_lowercase, 1), - ("has_digits", text::fn_has_digits, 1), - ("count_spaces", text::fn_count_spaces, 1), - ("count_uppercase", text::fn_count_uppercase, 1), - ("count_lowercase", text::fn_count_lowercase, 1), - ("count_chars", text::fn_count_chars, 1), - ("contains", text::fn_contains, 2), - ("contains_ignore_case", text::fn_contains_ignore_case, 2), - ("eq_ignore_case", text::fn_eq_ignore_case, 2), - ("starts_with", text::fn_starts_with, 2), - ("ends_with", text::fn_ends_with, 2), - ("lines", text::fn_lines, 1), - ("substring", text::fn_substring, 3), - ("strip_prefix", text::fn_strip_prefix, 2), - ("strip_suffix", text::fn_strip_suffix, 2), - ("split", text::fn_split, 2), - ("rsplit", text::fn_rsplit, 2), - ("split_once", text::fn_split_once, 2), - ("rsplit_once", text::fn_rsplit_once, 2), - ("split_words", text::fn_split_words, 1), -]; diff --git a/crates/utils/src/expr/functions/text.rs b/crates/utils/src/expr/functions/text.rs deleted file mode 100644 index 9cbce608..00000000 --- a/crates/utils/src/expr/functions/text.rs +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::borrow::Cow; - -use crate::expr::Variable; - -pub(crate) fn fn_trim(mut v: Vec) -> Variable { - v.remove(0).transform(|s| match s { - Cow::Borrowed(s) => Variable::from(s.trim()), - Cow::Owned(s) => Variable::from(s.trim().to_string()), - }) -} - -pub(crate) fn fn_trim_end(mut v: Vec) -> Variable { - v.remove(0).transform(|s| match s { - Cow::Borrowed(s) => Variable::from(s.trim_end()), - Cow::Owned(s) => Variable::from(s.trim_end().to_string()), - }) -} - -pub(crate) fn fn_trim_start(mut v: Vec) -> Variable { - v.remove(0).transform(|s| match s { - Cow::Borrowed(s) => Variable::from(s.trim_start()), - Cow::Owned(s) => Variable::from(s.trim_start().to_string()), - }) -} - -pub(crate) fn fn_len(v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.len(), - Variable::Array(a) => a.len(), - v => v.to_string().len(), - } - .into() -} - -pub(crate) fn fn_to_lowercase(mut v: Vec) -> Variable { - v.remove(0).transform(|s| Variable::from(s.to_lowercase())) -} - -pub(crate) fn fn_to_uppercase(mut v: Vec) -> Variable { - v.remove(0).transform(|s| Variable::from(s.to_uppercase())) -} - -pub(crate) fn fn_is_uppercase(mut v: Vec) -> Variable { - v.remove(0).transform(|s| { - s.chars() - .filter(|c| c.is_alphabetic()) - .all(|c| c.is_uppercase()) - .into() - }) -} - -pub(crate) fn fn_is_lowercase(mut v: Vec) -> Variable { - v.remove(0).transform(|s| { - s.chars() - .filter(|c| c.is_alphabetic()) - .all(|c| c.is_lowercase()) - .into() - }) -} - -pub(crate) fn fn_has_digits(mut v: Vec) -> Variable { - v.remove(0) - .transform(|s| s.chars().any(|c| c.is_ascii_digit()).into()) -} - -pub(crate) fn fn_split_words(v: Vec) -> Variable { - v[0].to_string() - .split_whitespace() - .filter(|word| word.chars().all(|c| c.is_alphanumeric())) - .map(|word| Variable::from(word.to_string())) - .collect::>() - .into() -} - -pub(crate) fn fn_count_spaces(v: Vec) -> Variable { - v[0].to_string() - .as_ref() - .chars() - .filter(|c| c.is_whitespace()) - .count() - .into() -} - -pub(crate) fn fn_count_uppercase(v: Vec) -> Variable { - v[0].to_string() - .as_ref() - .chars() - .filter(|c| c.is_alphabetic() && c.is_uppercase()) - .count() - .into() -} - -pub(crate) fn fn_count_lowercase(v: Vec) -> Variable { - v[0].to_string() - .as_ref() - .chars() - .filter(|c| c.is_alphabetic() && c.is_lowercase()) - .count() - .into() -} - -pub(crate) fn fn_count_chars(v: Vec) -> Variable { - v[0].to_string().as_ref().chars().count().into() -} - -pub(crate) fn fn_eq_ignore_case(v: Vec) -> Variable { - v[0].to_string() - .eq_ignore_ascii_case(v[1].to_string().as_ref()) - .into() -} - -pub(crate) fn fn_contains(v: Vec) -> Variable { - match &v[0] { - Variable::String(s) => s.contains(v[1].to_string().as_ref()), - Variable::Array(arr) => arr.contains(&v[1]), - val => val.to_string().contains(v[1].to_string().as_ref()), - } - .into() -} - -pub(crate) fn fn_contains_ignore_case(v: Vec) -> Variable { - let needle = v[1].to_string(); - match &v[0] { - Variable::String(s) => s.to_lowercase().contains(&needle.to_lowercase()), - Variable::Array(arr) => arr.iter().any(|v| match v { - Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()), - _ => false, - }), - val => val.to_string().contains(needle.as_ref()), - } - .into() -} - -pub(crate) fn fn_starts_with(v: Vec) -> Variable { - v[0].to_string() - .starts_with(v[1].to_string().as_ref()) - .into() -} - -pub(crate) fn fn_ends_with(v: Vec) -> Variable { - v[0].to_string().ends_with(v[1].to_string().as_ref()).into() -} - -pub(crate) fn fn_lines(mut v: Vec) -> Variable { - match v.remove(0) { - Variable::String(s) => s - .lines() - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into(), - val => val, - } -} - -pub(crate) fn fn_substring(v: Vec) -> Variable { - v[0].to_string() - .chars() - .skip(v[1].to_usize().unwrap_or_default()) - .take(v[2].to_usize().unwrap_or_default()) - .collect::() - .into() -} - -pub(crate) fn fn_strip_prefix(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap(); - let prefix = v.next().unwrap().into_string(); - - value.transform(|s| match s { - Cow::Borrowed(s) => s - .strip_prefix(prefix.as_ref()) - .map(Variable::from) - .unwrap_or_default(), - Cow::Owned(s) => s - .strip_prefix(prefix.as_ref()) - .map(|s| Variable::from(s.to_string())) - .unwrap_or_default(), - }) -} - -pub(crate) fn fn_strip_suffix(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap(); - let suffix = v.next().unwrap().into_string(); - - value.transform(|s| match s { - Cow::Borrowed(s) => s - .strip_suffix(suffix.as_ref()) - .map(Variable::from) - .unwrap_or_default(), - Cow::Owned(s) => s - .strip_suffix(suffix.as_ref()) - .map(|s| Variable::from(s.to_string())) - .unwrap_or_default(), - }) -} - -pub(crate) fn fn_split(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - - match value { - Cow::Borrowed(s) => s - .split(arg.as_ref()) - .map(Variable::from) - .collect::>() - .into(), - Cow::Owned(s) => s - .split(arg.as_ref()) - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into(), - } -} - -pub(crate) fn fn_rsplit(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - - match value { - Cow::Borrowed(s) => s - .rsplit(arg.as_ref()) - .map(Variable::from) - .collect::>() - .into(), - Cow::Owned(s) => s - .rsplit(arg.as_ref()) - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into(), - } -} - -pub(crate) fn fn_split_once(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - - match value { - Cow::Borrowed(s) => s - .split_once(arg.as_ref()) - .map(|(a, b)| Variable::Array(vec![Variable::from(a), Variable::from(b)])) - .unwrap_or_default(), - Cow::Owned(s) => s - .split_once(arg.as_ref()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::from(a.to_string()), - Variable::from(b.to_string()), - ]) - }) - .unwrap_or_default(), - } -} - -pub(crate) fn fn_rsplit_once(v: Vec) -> Variable { - let mut v = v.into_iter(); - let value = v.next().unwrap().into_string(); - let arg = v.next().unwrap().into_string(); - - match value { - Cow::Borrowed(s) => s - .rsplit_once(arg.as_ref()) - .map(|(a, b)| Variable::Array(vec![Variable::from(a), Variable::from(b)])) - .unwrap_or_default(), - Cow::Owned(s) => s - .rsplit_once(arg.as_ref()) - .map(|(a, b)| { - Variable::Array(vec![ - Variable::from(a.to_string()), - Variable::from(b.to_string()), - ]) - }) - .unwrap_or_default(), - } -} diff --git a/crates/utils/src/expr/mod.rs b/crates/utils/src/expr/mod.rs deleted file mode 100644 index 18db2710..00000000 --- a/crates/utils/src/expr/mod.rs +++ /dev/null @@ -1,308 +0,0 @@ -/* - * Copyright (c) 2020-2023, Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::borrow::Cow; - -use regex::Regex; - -use crate::config::utils::AsKey; - -use self::{parser::ExpressionParser, tokenizer::Tokenizer}; - -pub mod eval; -pub mod functions; -pub mod parser; -pub mod tokenizer; - -#[derive(Debug, PartialEq, Eq, Clone, Default)] -pub struct Expression { - pub items: Vec, -} - -#[derive(Debug, Clone)] -pub enum ExpressionItem { - Variable(u32), - Capture(u32), - Constant(Constant), - BinaryOperator(BinaryOperator), - UnaryOperator(UnaryOperator), - Regex(Regex), - JmpIf { val: bool, pos: u32 }, - Function { id: u32, num_args: u32 }, - ArrayAccess, - ArrayBuild(u32), -} - -#[derive(Debug)] -pub enum Variable<'x> { - String(Cow<'x, str>), - Integer(i64), - Float(f64), - Array(Vec>), -} - -impl Default for Variable<'_> { - fn default() -> Self { - Variable::Integer(0) - } -} - -#[derive(Debug, PartialEq, Clone)] -pub enum Constant { - Integer(i64), - Float(f64), - String(String), -} - -impl Eq for Constant {} - -impl From for Constant { - fn from(value: String) -> Self { - Constant::String(value) - } -} - -impl From for Constant { - fn from(value: bool) -> Self { - Constant::Integer(value as i64) - } -} - -impl From for Constant { - fn from(value: i64) -> Self { - Constant::Integer(value) - } -} - -impl From for Constant { - fn from(value: i32) -> Self { - Constant::Integer(value as i64) - } -} - -impl From for Constant { - fn from(value: i16) -> Self { - Constant::Integer(value as i64) - } -} - -impl From for Constant { - fn from(value: f64) -> Self { - Constant::Float(value) - } -} - -impl From for Constant { - fn from(value: usize) -> Self { - Constant::Integer(value as i64) - } -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum BinaryOperator { - Add, - Subtract, - Multiply, - Divide, - - And, - Or, - Xor, - - Eq, - Ne, - Lt, - Le, - Gt, - Ge, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum UnaryOperator { - Not, - Minus, -} - -#[derive(Debug, Clone)] -pub enum Token { - Variable(u32), - Capture(u32), - Function { - name: Cow<'static, str>, - id: u32, - num_args: u32, - }, - Constant(Constant), - Regex(Regex), - BinaryOperator(BinaryOperator), - UnaryOperator(UnaryOperator), - OpenParen, - CloseParen, - OpenBracket, - CloseBracket, - Comma, -} - -impl From for Variable<'_> { - fn from(value: usize) -> Self { - Variable::Integer(value as i64) - } -} - -impl From for Variable<'_> { - fn from(value: i64) -> Self { - Variable::Integer(value) - } -} - -impl From for Variable<'_> { - fn from(value: i32) -> Self { - Variable::Integer(value as i64) - } -} - -impl From for Variable<'_> { - fn from(value: i16) -> Self { - Variable::Integer(value as i64) - } -} - -impl From for Variable<'_> { - fn from(value: f64) -> Self { - Variable::Float(value) - } -} - -impl<'x> From<&'x str> for Variable<'x> { - fn from(value: &'x str) -> Self { - Variable::String(Cow::Borrowed(value)) - } -} - -impl From for Variable<'_> { - fn from(value: String) -> Self { - Variable::String(Cow::Owned(value)) - } -} - -impl<'x> From>> for Variable<'x> { - fn from(value: Vec>) -> Self { - Variable::Array(value) - } -} - -impl From for Variable<'_> { - fn from(value: bool) -> Self { - Variable::Integer(value as i64) - } -} - -impl Expression { - pub fn parse( - key: impl AsKey, - expr: &str, - token_map: impl Fn(&str) -> Result, - ) -> crate::config::Result { - ExpressionParser::new(Tokenizer::new(expr, token_map)) - .parse() - .map_err(|e| { - format!( - "Failed to parse expression {:?} for key {:?}: {}", - expr, - key.as_key(), - e - ) - }) - } -} - -impl> From for Expression { - fn from(value: T) -> Self { - Expression { - items: vec![ExpressionItem::Constant(value.into())], - } - } -} - -impl PartialEq for ExpressionItem { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Variable(l0), Self::Variable(r0)) => l0 == r0, - (Self::Constant(l0), Self::Constant(r0)) => l0 == r0, - (Self::BinaryOperator(l0), Self::BinaryOperator(r0)) => l0 == r0, - (Self::UnaryOperator(l0), Self::UnaryOperator(r0)) => l0 == r0, - (Self::Regex(_), Self::Regex(_)) => true, - ( - Self::JmpIf { - val: l_val, - pos: l_pos, - }, - Self::JmpIf { - val: r_val, - pos: r_pos, - }, - ) => l_val == r_val && l_pos == r_pos, - ( - Self::Function { - id: l_id, - num_args: l_num_args, - }, - Self::Function { - id: r_id, - num_args: r_num_args, - }, - ) => l_id == r_id && l_num_args == r_num_args, - (Self::ArrayBuild(l0), Self::ArrayBuild(r0)) => l0 == r0, - _ => core::mem::discriminant(self) == core::mem::discriminant(other), - } - } -} - -impl Eq for ExpressionItem {} - -impl PartialEq for Token { - fn eq(&self, other: &Self) -> bool { - match (self, other) { - (Self::Variable(l0), Self::Variable(r0)) => l0 == r0, - ( - Self::Function { - name: l_name, - id: l_id, - num_args: l_num_args, - }, - Self::Function { - name: r_name, - id: r_id, - num_args: r_num_args, - }, - ) => l_name == r_name && l_id == r_id && l_num_args == r_num_args, - (Self::Constant(l0), Self::Constant(r0)) => l0 == r0, - (Self::Regex(_), Self::Regex(_)) => true, - (Self::BinaryOperator(l0), Self::BinaryOperator(r0)) => l0 == r0, - (Self::UnaryOperator(l0), Self::UnaryOperator(r0)) => l0 == r0, - _ => core::mem::discriminant(self) == core::mem::discriminant(other), - } - } -} - -impl Eq for Token {} diff --git a/crates/utils/src/expr/parser.rs b/crates/utils/src/expr/parser.rs deleted file mode 100644 index 32730133..00000000 --- a/crates/utils/src/expr/parser.rs +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2020-2023, Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use super::{tokenizer::Tokenizer, BinaryOperator, Expression, ExpressionItem, Token}; - -pub struct ExpressionParser<'x, F> -where - F: Fn(&str) -> Result, -{ - pub(crate) tokenizer: Tokenizer<'x, F>, - pub(crate) output: Vec, - operator_stack: Vec<(Token, Option)>, - arg_count: Vec, -} - -pub(crate) const ID_ARRAY_ACCESS: u32 = u32::MAX; -pub(crate) const ID_ARRAY_BUILD: u32 = u32::MAX - 1; - -impl<'x, F> ExpressionParser<'x, F> -where - F: Fn(&str) -> Result, -{ - pub fn new(tokenizer: Tokenizer<'x, F>) -> Self { - Self { - tokenizer, - output: Vec::new(), - operator_stack: Vec::new(), - arg_count: Vec::new(), - } - } - - pub fn parse(mut self) -> Result { - let mut last_is_var_or_fnc = false; - - while let Some(token) = self.tokenizer.next()? { - let mut is_var_or_fnc = false; - match token { - Token::Variable(v) => { - self.inc_arg_count(); - is_var_or_fnc = true; - self.output.push(ExpressionItem::Variable(v)) - } - Token::Constant(c) => { - self.inc_arg_count(); - self.output.push(ExpressionItem::Constant(c)) - } - Token::Capture(c) => { - self.inc_arg_count(); - self.output.push(ExpressionItem::Capture(c)) - } - Token::UnaryOperator(uop) => { - self.operator_stack.push((Token::UnaryOperator(uop), None)) - } - Token::OpenParen => self.operator_stack.push((token, None)), - Token::CloseParen | Token::CloseBracket => { - let expect_token = if matches!(token, Token::CloseParen) { - Token::OpenParen - } else { - Token::OpenBracket - }; - loop { - match self.operator_stack.pop() { - Some((t, _)) if t == expect_token => { - break; - } - Some((Token::BinaryOperator(bop), jmp_pos)) => { - self.update_jmp_pos(jmp_pos); - self.output.push(ExpressionItem::BinaryOperator(bop)) - } - Some((Token::UnaryOperator(uop), _)) => { - self.output.push(ExpressionItem::UnaryOperator(uop)) - } - _ => return Err("Mismatched parentheses".to_string()), - } - } - - match self.operator_stack.last() { - Some((Token::Function { id, num_args, name }, _)) => { - let got_args = self.arg_count.pop().unwrap(); - if got_args != *num_args as i32 { - return Err(if *id != u32::MAX { - format!( - "Expression function {:?} expected {} arguments, got {}", - name, num_args, got_args - ) - } else { - "Missing array index".to_string() - }); - } - - let expr = match *id { - ID_ARRAY_ACCESS => ExpressionItem::ArrayAccess, - ID_ARRAY_BUILD => ExpressionItem::ArrayBuild(*num_args), - id => ExpressionItem::Function { - id, - num_args: *num_args, - }, - }; - - self.operator_stack.pop(); - self.output.push(expr); - } - Some((Token::Regex(regex), _)) => { - if self.arg_count.pop().unwrap() != 1 { - return Err("Expression function \"matches\" expected 2 arguments" - .to_string()); - } - self.output.push(ExpressionItem::Regex(regex.clone())); - self.operator_stack.pop(); - } - _ => {} - } - - is_var_or_fnc = true; - } - Token::BinaryOperator(bop) => { - self.dec_arg_count(); - while let Some((top_token, prev_jmp_pos)) = self.operator_stack.last() { - match top_token { - Token::BinaryOperator(top_bop) => { - if bop.precedence() <= top_bop.precedence() { - let top_bop = *top_bop; - let jmp_pos = *prev_jmp_pos; - self.update_jmp_pos(jmp_pos); - self.operator_stack.pop(); - self.output.push(ExpressionItem::BinaryOperator(top_bop)); - } else { - break; - } - } - Token::UnaryOperator(top_uop) => { - let top_uop = *top_uop; - self.operator_stack.pop(); - self.output.push(ExpressionItem::UnaryOperator(top_uop)); - } - _ => break, - } - } - - // Add jump instruction for short-circuiting - let jmp_pos = match bop { - BinaryOperator::And => { - self.output - .push(ExpressionItem::JmpIf { val: false, pos: 0 }); - Some(self.output.len() - 1) - } - BinaryOperator::Or => { - self.output - .push(ExpressionItem::JmpIf { val: true, pos: 0 }); - Some(self.output.len() - 1) - } - _ => None, - }; - - self.operator_stack - .push((Token::BinaryOperator(bop), jmp_pos)); - } - Token::Function { id, name, num_args } => { - self.inc_arg_count(); - self.arg_count.push(0); - self.operator_stack - .push((Token::Function { id, name, num_args }, None)) - } - Token::Regex(regex) => { - self.inc_arg_count(); - self.arg_count.push(0); - self.operator_stack.push((Token::Regex(regex), None)) - } - Token::OpenBracket => { - // Array functions - let (id, num_args, arg_count) = if last_is_var_or_fnc { - (ID_ARRAY_ACCESS, 2, 1) - } else { - self.inc_arg_count(); - (ID_ARRAY_BUILD, 0, 0) - }; - self.arg_count.push(arg_count); - self.operator_stack.push(( - Token::Function { - id, - name: "array".into(), - num_args, - }, - None, - )); - self.operator_stack.push((token, None)); - } - Token::Comma => { - while let Some((token, jmp_pos)) = self.operator_stack.last() { - match token { - Token::OpenParen => break, - Token::BinaryOperator(bop) => { - let bop = *bop; - let jmp_pos = *jmp_pos; - self.update_jmp_pos(jmp_pos); - self.output.push(ExpressionItem::BinaryOperator(bop)); - self.operator_stack.pop(); - } - Token::UnaryOperator(uop) => { - self.output.push(ExpressionItem::UnaryOperator(*uop)); - self.operator_stack.pop(); - } - _ => break, - } - } - } - } - last_is_var_or_fnc = is_var_or_fnc; - } - - while let Some((token, jmp_pos)) = self.operator_stack.pop() { - match token { - Token::BinaryOperator(bop) => { - self.update_jmp_pos(jmp_pos); - self.output.push(ExpressionItem::BinaryOperator(bop)) - } - Token::UnaryOperator(uop) => self.output.push(ExpressionItem::UnaryOperator(uop)), - _ => return Err("Invalid token on the operator stack".to_string()), - } - } - - if self.operator_stack.is_empty() { - Ok(Expression { items: self.output }) - } else { - Err("Invalid expression".to_string()) - } - } - - fn inc_arg_count(&mut self) { - if let Some(x) = self.arg_count.last_mut() { - *x = x.saturating_add(1); - let op_pos = self.operator_stack.len().saturating_sub(2); - match self.operator_stack.get_mut(op_pos) { - Some((Token::Function { num_args, id, .. }, _)) if *id == ID_ARRAY_BUILD => { - *num_args += 1; - } - _ => {} - } - } - } - - fn dec_arg_count(&mut self) { - if let Some(x) = self.arg_count.last_mut() { - *x = x.saturating_sub(1); - } - } - - fn update_jmp_pos(&mut self, jmp_pos: Option) { - if let Some(jmp_pos) = jmp_pos { - let cur_pos = self.output.len(); - if let ExpressionItem::JmpIf { pos, .. } = &mut self.output[jmp_pos] { - *pos = (cur_pos - jmp_pos) as u32; - } else { - #[cfg(test)] - panic!("Invalid jump position"); - } - } - } -} - -impl BinaryOperator { - fn precedence(&self) -> i32 { - match self { - BinaryOperator::Multiply | BinaryOperator::Divide => 7, - BinaryOperator::Add | BinaryOperator::Subtract => 6, - BinaryOperator::Gt | BinaryOperator::Ge | BinaryOperator::Lt | BinaryOperator::Le => 5, - BinaryOperator::Eq | BinaryOperator::Ne => 4, - BinaryOperator::Xor => 3, - BinaryOperator::And => 2, - BinaryOperator::Or => 1, - } - } -} diff --git a/crates/utils/src/expr/tokenizer.rs b/crates/utils/src/expr/tokenizer.rs deleted file mode 100644 index cb08cf6a..00000000 --- a/crates/utils/src/expr/tokenizer.rs +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Copyright (c) 2020-2023, Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{borrow::Cow, iter::Peekable, slice::Iter}; - -use regex::Regex; -use smtp_proto::IntoString; - -use super::{functions::FUNCTIONS, BinaryOperator, Constant, Token, UnaryOperator}; - -pub struct Tokenizer<'x, F> -where - F: Fn(&str) -> Result, -{ - pub(crate) iter: Peekable>, - token_map: F, - buf: Vec, - depth: u32, - next_token: Vec, - has_number: bool, - has_dot: bool, - has_alpha: bool, - is_start: bool, - is_eof: bool, -} - -impl<'x, F> Tokenizer<'x, F> -where - F: Fn(&str) -> Result, -{ - #[allow(clippy::should_implement_trait)] - pub fn new(expr: &'x str, token_map: F) -> Self { - Self { - iter: expr.as_bytes().iter().peekable(), - buf: Vec::new(), - depth: 0, - next_token: Vec::with_capacity(2), - has_number: false, - has_dot: false, - has_alpha: false, - is_start: true, - is_eof: false, - token_map, - } - } - - #[allow(clippy::should_implement_trait)] - pub fn next(&mut self) -> Result, String> { - if let Some(token) = self.next_token.pop() { - return Ok(Some(token)); - } else if self.is_eof { - return Ok(None); - } - - while let Some(&ch) = self.iter.next() { - match ch { - b'A'..=b'Z' | b'a'..=b'z' | b'_' | b'$' => { - self.buf.push(ch); - self.has_alpha = true; - } - b'0'..=b'9' => { - self.buf.push(ch); - self.has_number = true; - } - b'.' => { - self.buf.push(ch); - self.has_dot = true; - } - b'}' => { - self.is_eof = true; - break; - } - b'-' if self.buf.last().map_or(false, |c| *c == b'[') => { - self.buf.push(ch); - } - b':' if self.buf.contains(&b'.') => { - self.buf.push(ch); - } - b']' if self.buf.contains(&b'[') => { - self.buf.push(b']'); - } - b'*' if self.buf.last().map_or(false, |&c| c == b'[' || c == b'.') => { - self.buf.push(ch); - } - _ => { - let (prev_token, ch) = if ch == b'(' && self.buf.eq(b"matches") { - // Parse regular expressions - let stop_ch = self.find_char(&[b'\"', b'\''])?; - let regex_str = self.parse_string(stop_ch)?; - let regex = Regex::new(®ex_str).map_err(|e| { - format!("Invalid regular expression {:?}: {}", regex_str, e) - })?; - self.has_alpha = false; - self.buf.clear(); - self.find_char(&[b','])?; - (Token::Regex(regex).into(), b'(') - } else if !self.buf.is_empty() { - self.is_start = false; - (self.parse_buf()?.into(), ch) - } else { - (None, ch) - }; - let token = match ch { - b'&' => { - if matches!(self.iter.peek(), Some(b'&')) { - self.iter.next(); - } - Token::BinaryOperator(BinaryOperator::And) - } - b'|' => { - if matches!(self.iter.peek(), Some(b'|')) { - self.iter.next(); - } - Token::BinaryOperator(BinaryOperator::Or) - } - b'!' => { - if matches!(self.iter.peek(), Some(b'=')) { - self.iter.next(); - Token::BinaryOperator(BinaryOperator::Ne) - } else { - Token::UnaryOperator(UnaryOperator::Not) - } - } - b'^' => Token::BinaryOperator(BinaryOperator::Xor), - b'(' => { - self.depth += 1; - Token::OpenParen - } - b')' => { - if self.depth == 0 { - return Err("Unmatched close parenthesis".to_string()); - } - self.depth -= 1; - Token::CloseParen - } - b'+' => Token::BinaryOperator(BinaryOperator::Add), - b'*' => Token::BinaryOperator(BinaryOperator::Multiply), - b'/' => Token::BinaryOperator(BinaryOperator::Divide), - b'-' => { - if self.is_start { - Token::UnaryOperator(UnaryOperator::Minus) - } else { - Token::BinaryOperator(BinaryOperator::Subtract) - } - } - b'=' => match self.iter.next() { - Some(b'=') => Token::BinaryOperator(BinaryOperator::Eq), - Some(b'>') => Token::BinaryOperator(BinaryOperator::Ge), - Some(b'<') => Token::BinaryOperator(BinaryOperator::Le), - _ => Token::BinaryOperator(BinaryOperator::Eq), - }, - b'>' => match self.iter.peek() { - Some(b'=') => { - self.iter.next(); - Token::BinaryOperator(BinaryOperator::Ge) - } - _ => Token::BinaryOperator(BinaryOperator::Gt), - }, - b'<' => match self.iter.peek() { - Some(b'=') => { - self.iter.next(); - Token::BinaryOperator(BinaryOperator::Le) - } - _ => Token::BinaryOperator(BinaryOperator::Lt), - }, - b',' => Token::Comma, - b'[' => Token::OpenBracket, - b']' => Token::CloseBracket, - b' ' | b'\r' | b'\n' => { - if prev_token.is_some() { - return Ok(prev_token); - } else { - continue; - } - } - b'\"' | b'\'' => Token::Constant(Constant::String(self.parse_string(ch)?)), - _ => { - return Err(format!("Invalid character {:?}", char::from(ch),)); - } - }; - self.is_start = matches!( - token, - Token::OpenParen | Token::Comma | Token::BinaryOperator(_) - ); - - return if prev_token.is_some() { - self.next_token.push(token); - Ok(prev_token) - } else { - Ok(Some(token)) - }; - } - } - } - - if self.depth > 0 { - Err("Unmatched open parenthesis".to_string()) - } else if !self.buf.is_empty() { - self.parse_buf().map(Some) - } else { - Ok(None) - } - } - - fn find_char(&mut self, chars: &[u8]) -> Result { - for &ch in self.iter.by_ref() { - if !ch.is_ascii_whitespace() { - return if chars.contains(&ch) { - Ok(ch) - } else { - Err(format!( - "Expected {:?}, found invalid character {:?}", - char::from(chars[0]), - char::from(ch), - )) - }; - } - } - - Err("Unexpected end of expression".to_string()) - } - - fn parse_string(&mut self, stop_ch: u8) -> Result { - let mut buf = Vec::with_capacity(16); - let mut last_ch = 0; - let mut found_end = false; - - for &ch in self.iter.by_ref() { - if last_ch != b'\\' { - if ch != stop_ch { - buf.push(ch); - } else { - found_end = true; - break; - } - } else { - match ch { - b'n' => { - buf.push(b'\n'); - } - b'r' => { - buf.push(b'\r'); - } - b't' => { - buf.push(b'\t'); - } - _ => { - buf.push(ch); - } - } - } - - last_ch = ch; - } - - if found_end { - String::from_utf8(buf).map_err(|_| "Invalid UTF-8".to_string()) - } else { - Err("Unterminated string".to_string()) - } - } - - fn parse_buf(&mut self) -> Result { - let buf = std::mem::take(&mut self.buf).into_string(); - if self.has_number && !self.has_alpha { - self.has_number = false; - if self.has_dot { - self.has_dot = false; - - buf.parse::() - .map(|f| Token::Constant(Constant::Float(f))) - .map_err(|_| format!("Invalid float value {}", buf,)) - } else { - buf.parse::() - .map(|i| Token::Constant(Constant::Integer(i))) - .map_err(|_| format!("Invalid integer value {}", buf,)) - } - } else { - let has_dot = self.has_dot; - let has_number = self.has_number; - - self.has_alpha = false; - self.has_number = false; - self.has_dot = false; - - if !has_number && !has_dot && [4, 5].contains(&buf.len()) { - if buf == "true" { - return Ok(Token::Constant(Constant::Integer(1))); - } else if buf == "false" { - return Ok(Token::Constant(Constant::Integer(0))); - } - } - - if let Some(regex_capture) = buf.strip_prefix('$').and_then(|v| v.parse::().ok()) { - Ok(Token::Capture(regex_capture)) - } else if let Some((idx, (name, _, num_args))) = FUNCTIONS - .iter() - .enumerate() - .find(|(_, (name, _, _))| name == &buf) - { - Ok(Token::Function { - name: Cow::Borrowed(*name), - id: idx as u32, - num_args: *num_args, - }) - } else { - (self.token_map)(&buf).map(|t| match t { - Token::Function { name, id, num_args } => Token::Function { - name, - id: id + FUNCTIONS.len() as u32, - num_args, - }, - t => t, - }) - } - } - } -} diff --git a/crates/utils/src/ipc.rs b/crates/utils/src/ipc.rs deleted file mode 100644 index b36d810a..00000000 --- a/crates/utils/src/ipc.rs +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::borrow::Cow; - -use tokio::sync::oneshot; - -use crate::BlobHash; - -#[derive(Debug)] -pub enum DeliveryEvent { - Ingest { - message: IngestMessage, - result_tx: oneshot::Sender>, - }, - Stop, -} - -#[derive(Debug)] -pub struct IngestMessage { - pub sender_address: String, - pub recipients: Vec, - pub message_blob: BlobHash, - pub message_size: usize, -} - -#[derive(Debug, Clone)] -pub enum DeliveryResult { - Success, - TemporaryFailure { - reason: Cow<'static, str>, - }, - PermanentFailure { - code: [u8; 3], - reason: Cow<'static, str>, - }, -} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 9b732101..851359ff 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -21,36 +21,21 @@ * for more details. */ -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; -use config::Config; - -pub mod acme; pub mod codec; pub mod config; -pub mod expr; -pub mod ipc; -pub mod listener; pub mod lru_cache; pub mod map; pub mod snowflake; pub mod suffixlist; pub mod url_params; -use opentelemetry::KeyValue; -use opentelemetry_otlp::WithExportConfig; -use opentelemetry_sdk::{ - trace::{self, Sampler}, - Resource, -}; -use opentelemetry_semantic_conventions::resource::{SERVICE_NAME, SERVICE_VERSION}; use rustls::{ client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, ClientConfig, RootCertStore, SignatureScheme, }; use rustls_pki_types::TrustAnchor; -use tracing_appender::non_blocking::WorkerGuard; -use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, EnvFilter}; pub const BLOB_HASH_LEN: usize = 32; @@ -156,124 +141,6 @@ pub fn failed(message: &str) -> ! { std::process::exit(1); } -pub fn enable_tracing(config: &Config, message: &str) -> config::Result> { - let level = config.value("global.tracing.level").unwrap_or("info"); - let env_filter = EnvFilter::builder() - .parse(format!( - "smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level}" - )) - .failed("Failed to log level"); - let result = match config.value("global.tracing.method").unwrap_or_default() { - "log" => { - let path = config.value_require("global.tracing.path")?; - let prefix = config.value_require("global.tracing.prefix")?; - let file_appender = match config.value("global.tracing.rotate").unwrap_or("daily") { - "daily" => tracing_appender::rolling::daily(path, prefix), - "hourly" => tracing_appender::rolling::hourly(path, prefix), - "minutely" => tracing_appender::rolling::minutely(path, prefix), - "never" => tracing_appender::rolling::never(path, prefix), - rotate => { - return Err(format!("Unsupported log rotation strategy {rotate:?}")); - } - }; - - let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); - tracing::subscriber::set_global_default( - tracing_subscriber::FmtSubscriber::builder() - .with_env_filter(env_filter) - .with_writer(non_blocking) - .with_ansi(config.property_or_default("global.tracing.ansi", "true")?) - .finish(), - ) - .failed("Failed to set subscriber"); - Ok(guard.into()) - } - "stdout" => { - tracing::subscriber::set_global_default( - tracing_subscriber::FmtSubscriber::builder() - .with_env_filter(env_filter) - .with_ansi(config.property_or_default("global.tracing.ansi", "true")?) - .finish(), - ) - .failed("Failed to set subscriber"); - - Ok(None) - } - "otel" | "open-telemetry" => { - let tracer = match config.value_require("global.tracing.transport")? { - "grpc" => { - let mut exporter = opentelemetry_otlp::new_exporter().tonic(); - if let Some(endpoint) = config.value("global.tracing.endpoint") { - exporter = exporter.with_endpoint(endpoint); - } - opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter(exporter) - } - "http" => { - let mut headers = HashMap::new(); - for (_, value) in config.values("global.tracing.headers") { - if let Some((key, value)) = value.split_once(':') { - headers.insert(key.trim().to_string(), value.trim().to_string()); - } else { - return Err(format!("Invalid open-telemetry header {value:?}")); - } - } - let mut exporter = opentelemetry_otlp::new_exporter() - .http() - .with_endpoint(config.value_require("global.tracing.endpoint")?); - if !headers.is_empty() { - exporter = exporter.with_headers(headers); - } - opentelemetry_otlp::new_pipeline() - .tracing() - .with_exporter(exporter) - } - transport => { - return Err(format!( - "Unsupported open-telemetry transport {transport:?}" - )); - } - } - .with_trace_config( - trace::config() - .with_resource(Resource::new(vec![ - KeyValue::new(SERVICE_NAME, "stalwart-smtp".to_string()), - KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION").to_string()), - ])) - .with_sampler(Sampler::AlwaysOn), - ) - .install_batch(opentelemetry_sdk::runtime::Tokio) - .failed("Failed to create tracer"); - - tracing::subscriber::set_global_default( - tracing_subscriber::Registry::default() - .with(tracing_opentelemetry::layer().with_tracer(tracer)) - .with(env_filter), - ) - .failed("Failed to set subscriber"); - - Ok(None) - } - #[cfg(unix)] - "journal" => { - tracing::subscriber::set_global_default( - tracing_subscriber::Registry::default() - .with(tracing_journald::layer().failed("Failed to configure journal")) - .with(env_filter), - ) - .failed("Failed to set subscriber"); - - Ok(None) - } - _ => Ok(None), - }; - - tracing::info!(message); - - result -} - pub async fn wait_for_shutdown(message: &str) { #[cfg(not(target_env = "msvc"))] { diff --git a/crates/utils/src/listener/limiter.rs b/crates/utils/src/listener/limiter.rs deleted file mode 100644 index 592d1c56..00000000 --- a/crates/utils/src/listener/limiter.rs +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, - time::SystemTime, -}; - -use crate::config::Rate; - -#[derive(Debug)] -pub struct RateLimiter { - next_refill: AtomicU64, - used_tokens: AtomicU64, -} - -#[derive(Debug, Clone)] -pub struct ConcurrencyLimiter { - pub max_concurrent: u64, - pub concurrent: Arc, -} - -#[derive(Default)] -pub struct InFlight { - concurrent: Arc, -} - -impl Drop for InFlight { - fn drop(&mut self) { - self.concurrent.fetch_sub(1, Ordering::Relaxed); - } -} - -impl RateLimiter { - pub fn new(rate: &Rate) -> Self { - RateLimiter { - next_refill: (now() + rate.period.as_secs()).into(), - used_tokens: 0.into(), - } - } - - pub fn is_allowed(&self, rate: &Rate) -> bool { - // Check rate limit - if self.used_tokens.fetch_add(1, Ordering::Relaxed) < rate.requests { - true - } else { - let now = now(); - if self.next_refill.load(Ordering::Relaxed) <= now { - self.next_refill - .store(now + rate.period.as_secs(), Ordering::Relaxed); - self.used_tokens.store(1, Ordering::Relaxed); - true - } else { - false - } - } - } - - pub fn is_allowed_soft(&self, rate: &Rate) -> bool { - self.used_tokens.load(Ordering::Relaxed) < rate.requests - || self.next_refill.load(Ordering::Relaxed) <= now() - } - - pub fn secs_to_refill(&self) -> u64 { - self.next_refill - .load(Ordering::Relaxed) - .saturating_sub(now()) - } - - pub fn is_active(&self) -> bool { - self.next_refill.load(Ordering::Relaxed) > now() - } -} - -impl ConcurrencyLimiter { - pub fn new(max_concurrent: u64) -> Self { - ConcurrencyLimiter { - max_concurrent, - concurrent: Arc::new(0.into()), - } - } - - pub fn is_allowed(&self) -> Option { - if self.concurrent.load(Ordering::Relaxed) < self.max_concurrent { - // Return in-flight request - self.concurrent.fetch_add(1, Ordering::Relaxed); - Some(InFlight { - concurrent: self.concurrent.clone(), - }) - } else { - None - } - } - - pub fn check_is_allowed(&self) -> bool { - self.concurrent.load(Ordering::Relaxed) < self.max_concurrent - } - - pub fn is_active(&self) -> bool { - self.concurrent.load(Ordering::Relaxed) > 0 - } -} - -impl InFlight { - pub fn num_concurrent(&self) -> u64 { - self.concurrent.load(Ordering::Relaxed) - } -} - -fn now() -> u64 { - SystemTime::UNIX_EPOCH - .elapsed() - .unwrap_or_default() - .as_secs() -} diff --git a/crates/utils/src/listener/listen.rs b/crates/utils/src/listener/listen.rs deleted file mode 100644 index be516f93..00000000 --- a/crates/utils/src/listener/listen.rs +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - net::{IpAddr, SocketAddr}, - sync::Arc, - time::Duration, -}; - -use proxy_header::io::ProxiedStream; -use rustls::crypto::ring::cipher_suite::TLS13_AES_128_GCM_SHA256; -use tokio::{ - net::{TcpListener, TcpStream}, - sync::watch, -}; -use tokio_rustls::server::TlsStream; -use tracing::Span; - -use crate::{ - acme::SpawnAcme, - config::{Config, Listener, Server, ServerProtocol, Servers}, - failed, - listener::SessionData, - UnwrapFailure, -}; - -use super::{ - limiter::ConcurrencyLimiter, ServerInstance, SessionManager, SessionStream, TcpAcceptorResult, -}; - -impl Server { - pub fn spawn(self, manager: impl SessionManager, shutdown_rx: watch::Receiver) { - // Prepare instance - let instance = Arc::new(ServerInstance { - data: if matches!(self.protocol, ServerProtocol::Smtp | ServerProtocol::Lmtp) { - format!("220 {} {}\r\n", self.hostname, self.data) - } else { - self.data - }, - id: self.id, - listener_id: self.internal_id, - protocol: self.protocol, - hostname: self.hostname, - acceptor: self.acceptor, - proxy_networks: self.proxy_networks, - limiter: ConcurrencyLimiter::new(self.max_connections), - shutdown_rx, - }); - let is_tls = self.tls_implicit; - let has_proxies = !instance.proxy_networks.is_empty(); - - // Spawn listeners - for listener in self.listeners { - tracing::info!( - id = instance.id, - protocol = ?instance.protocol, - bind.ip = listener.addr.ip().to_string(), - bind.port = listener.addr.port(), - tls = is_tls, - "Starting listener" - ); - let local_ip = listener.addr.ip(); - - // Obtain TCP options - let opts = SocketOpts { - nodelay: listener.nodelay, - ttl: listener.ttl, - linger: listener.linger, - }; - - // Bind socket - let listener = listener.listen(); - - // Spawn listener - let mut shutdown_rx = instance.shutdown_rx.clone(); - let manager = manager.clone(); - let instance = instance.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - stream = listener.accept() => { - match stream { - Ok((stream, remote_addr)) => { - if has_proxies && instance.proxy_networks.iter().any(|network| network.matches(&remote_addr.ip())) { - let instance = instance.clone(); - let manager = manager.clone(); - - // Set socket options - opts.apply(&stream); - - tokio::spawn(async move { - match ProxiedStream::create_from_tokio(stream, Default::default()).await { - Ok(stream) =>{ - let remote_addr = stream.proxy_header() - .proxied_address() - .map(|addr| addr.source) - .unwrap_or(remote_addr); - if let Some(session) = instance.build_session(stream, local_ip, remote_addr, &manager) { - // Spawn session - manager.spawn(session, is_tls); - } - } - Err(err) => { - tracing::trace!(context = "io", - event = "error", - instance = instance.id, - protocol = ?instance.protocol, - reason = %err, - "Failed to accept proxied TCP connection"); - } - } - }); - } else if let Some(session) = instance.build_session(stream, local_ip, remote_addr, &manager) { - // Set socket options - opts.apply(&session.stream); - - // Spawn session - manager.spawn(session, is_tls); - } - } - Err(err) => { - tracing::trace!(context = "io", - event = "error", - instance = instance.id, - protocol = ?instance.protocol, - "Failed to accept TCP connection: {}", err); - } - } - }, - _ = shutdown_rx.changed() => { - tracing::debug!( - event = "shutdown", - instance = instance.id, - protocol = ?instance.protocol, - "Listener shutting down."); - manager.shutdown().await; - break; - } - }; - } - }); - } - } -} - -trait BuildSession { - fn build_session( - &self, - stream: T, - local_ip: IpAddr, - remote_addr: SocketAddr, - manager: &M, - ) -> Option>; -} - -impl BuildSession for Arc { - fn build_session( - &self, - stream: T, - local_ip: IpAddr, - remote_addr: SocketAddr, - manager: &M, - ) -> Option> { - // Convert mapped IPv6 addresses to IPv4 - let remote_ip = match remote_addr.ip() { - IpAddr::V6(ip) => ip - .to_ipv4_mapped() - .map(IpAddr::V4) - .unwrap_or(IpAddr::V6(ip)), - remote_ip => remote_ip, - }; - let remote_port = remote_addr.port(); - - // Check if blocked - if manager.is_ip_blocked(&remote_ip) { - tracing::debug!( - context = "listener", - event = "blocked", - instance = self.id, - protocol = ?self.protocol, - remote.ip = remote_ip.to_string(), - remote.port = remote_port, - "Dropping connection from blocked IP." - ); - None - } else if let Some(in_flight) = self.limiter.is_allowed() { - // Enforce concurrency - SessionData { - stream, - in_flight, - span: tracing::info_span!( - "session", - instance = self.id, - protocol = ?self.protocol, - remote.ip = remote_ip.to_string(), - remote.port = remote_port, - ), - local_ip, - remote_ip, - remote_port, - instance: self.clone(), - } - .into() - } else { - tracing::info!( - context = "throttle", - event = "too-many-requests", - instance = self.id, - protocol = ?self.protocol, - remote.ip = remote_ip.to_string(), - remote.port = remote_port, - max_concurrent = self.limiter.max_concurrent, - "Too many concurrent connections." - ); - None - } - } -} - -pub struct SocketOpts { - pub nodelay: bool, - pub ttl: Option, - pub linger: Option, -} - -impl SocketOpts { - pub fn apply(&self, stream: &TcpStream) { - // Set TCP options - if let Err(err) = stream.set_nodelay(self.nodelay) { - tracing::warn!( - context = "tcp", - event = "error", - "Failed to set no-delay: {}", - err - ); - } - if let Some(ttl) = self.ttl { - if let Err(err) = stream.set_ttl(ttl) { - tracing::warn!( - context = "tcp", - event = "error", - "Failed to set TTL: {}", - err - ); - } - } - if self.linger.is_some() { - if let Err(err) = stream.set_linger(self.linger) { - tracing::warn!( - context = "tcp", - event = "error", - "Failed to set linger: {}", - err - ); - } - } - } -} - -impl Servers { - pub fn bind(&self, config: &Config) { - // Bind as root - for server in &self.inner { - for listener in &server.listeners { - listener - .socket - .bind(listener.addr) - .failed(&format!("Failed to bind to {}", listener.addr)); - } - } - - // Drop privileges - #[cfg(not(target_env = "msvc"))] - { - if let Some(run_as_user) = config.value("server.run-as.user") { - let mut pd = privdrop::PrivDrop::default().user(run_as_user); - if let Some(run_as_group) = config.value("server.run-as.group") { - pd = pd.group(run_as_group); - } - pd.apply().failed("Failed to drop privileges"); - } - } - } - - pub fn spawn( - self, - spawn: impl Fn(Server, watch::Receiver), - ) -> (watch::Sender, watch::Receiver) { - // Spawn listeners - let (shutdown_tx, shutdown_rx) = watch::channel(false); - for server in self.inner { - spawn(server, shutdown_rx.clone()); - } - - // Spawn ACME managers - for acme_manager in self.acme_managers { - acme_manager.spawn(shutdown_rx.clone()); - } - - (shutdown_tx, shutdown_rx) - } -} - -impl Listener { - pub fn listen(self) -> TcpListener { - self.socket - .listen(self.backlog.unwrap_or(1024)) - .unwrap_or_else(|err| failed(&format!("Failed to listen on {}: {}", self.addr, err))) - } -} - -impl ServerInstance { - pub async fn tls_accept( - &self, - stream: T, - span: &Span, - ) -> Result, ()> { - match self.acceptor.accept(stream).await { - TcpAcceptorResult::Tls(accept) => match accept.await { - Ok(stream) => { - tracing::info!( - parent: span, - context = "tls", - event = "handshake", - version = ?stream.get_ref().1.protocol_version().unwrap_or(rustls::ProtocolVersion::TLSv1_3), - cipher = ?stream.get_ref().1.negotiated_cipher_suite().unwrap_or(TLS13_AES_128_GCM_SHA256), - ); - Ok(stream) - } - Err(err) => { - tracing::debug!( - parent: span, - context = "tls", - event = "error", - "Failed to accept TLS connection: {}", - err - ); - Err(()) - } - }, - TcpAcceptorResult::Plain(_) | TcpAcceptorResult::Close => { - tracing::debug!( - parent: span, - context = "tls", - event = "error", - "Failed to accept TLS connection: {}", - "TLS is not configured for this server." - ); - Err(()) - } - } - } -} diff --git a/crates/utils/src/listener/mod.rs b/crates/utils/src/listener/mod.rs deleted file mode 100644 index ad9f41cd..00000000 --- a/crates/utils/src/listener/mod.rs +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{borrow::Cow, net::IpAddr, sync::Arc}; - -use crate::{ - acme::AcmeManager, - config::{ipmask::IpAddrMask, ServerProtocol}, -}; -use rustls::ServerConfig; -use std::fmt::Debug; -use tokio::{ - io::{AsyncRead, AsyncWrite}, - sync::watch, -}; -use tokio_rustls::{Accept, TlsAcceptor}; - -use self::limiter::{ConcurrencyLimiter, InFlight}; - -pub mod limiter; -pub mod listen; -pub mod stream; -pub mod tls; - -pub struct ServerInstance { - pub id: String, - pub listener_id: u16, - pub protocol: ServerProtocol, - pub hostname: String, - pub data: String, - pub acceptor: TcpAcceptor, - pub limiter: ConcurrencyLimiter, - pub proxy_networks: Vec, - pub shutdown_rx: watch::Receiver, -} - -#[derive(Default)] -pub enum TcpAcceptor { - Tls(TlsAcceptor), - Acme { - challenge: Arc, - default: Arc, - manager: Arc, - }, - #[default] - Plain, -} - -#[allow(clippy::large_enum_variant)] -pub enum TcpAcceptorResult -where - IO: AsyncRead + AsyncWrite + Unpin, -{ - Tls(Accept), - Plain(IO), - Close, -} - -pub struct SessionData { - pub stream: T, - pub local_ip: IpAddr, - pub remote_ip: IpAddr, - pub remote_port: u16, - pub span: tracing::Span, - pub in_flight: InFlight, - pub instance: Arc, -} - -pub trait SessionStream: AsyncRead + AsyncWrite + Unpin + 'static + Sync + Send { - fn is_tls(&self) -> bool; - fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>); -} - -pub trait SessionManager: Sync + Send + 'static + Clone { - fn spawn(&self, mut session: SessionData, is_tls: bool) { - let manager = self.clone(); - - tokio::spawn(async move { - if is_tls { - match session.instance.acceptor.accept(session.stream).await { - TcpAcceptorResult::Tls(accept) => match accept.await { - Ok(stream) => { - let session = SessionData { - stream, - local_ip: session.local_ip, - remote_ip: session.remote_ip, - remote_port: session.remote_port, - span: session.span, - in_flight: session.in_flight, - instance: session.instance, - }; - manager.handle(session).await; - } - Err(err) => { - tracing::debug!( - context = "tls", - event = "error", - instance = session.instance.id, - protocol = ?session.instance.protocol, - remote.ip = session.remote_ip.to_string(), - "Failed to accept TLS connection: {}", - err - ); - } - }, - TcpAcceptorResult::Plain(stream) => { - session.stream = stream; - manager.handle(session).await; - } - TcpAcceptorResult::Close => (), - } - } else { - manager.handle(session).await; - } - }); - } - - fn handle( - self, - session: SessionData, - ) -> impl std::future::Future + Send; - fn is_ip_blocked(&self, addr: &IpAddr) -> bool; - - fn shutdown(&self) -> impl std::future::Future + Send; -} - -impl Debug for TcpAcceptor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Tls(_) => f.debug_tuple("Tls").finish(), - Self::Acme { - challenge, - default, - manager, - } => f - .debug_struct("Acme") - .field("challenge", challenge) - .field("default", default) - .field("manager", manager) - .finish(), - Self::Plain => write!(f, "Plain"), - } - } -} diff --git a/crates/utils/src/listener/stream.rs b/crates/utils/src/listener/stream.rs deleted file mode 100644 index d01e7174..00000000 --- a/crates/utils/src/listener/stream.rs +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::borrow::Cow; - -use proxy_header::io::ProxiedStream; -use tokio::{ - io::{AsyncRead, AsyncWrite}, - net::TcpStream, -}; -use tokio_rustls::server::TlsStream; - -use super::SessionStream; - -impl SessionStream for TcpStream { - fn is_tls(&self) -> bool { - false - } - - fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>) { - (Cow::Borrowed(""), Cow::Borrowed("")) - } -} - -impl SessionStream for TlsStream { - fn is_tls(&self) -> bool { - true - } - - fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>) { - let (_, conn) = self.get_ref(); - - ( - match conn - .protocol_version() - .unwrap_or(rustls::ProtocolVersion::Unknown(0)) - { - rustls::ProtocolVersion::SSLv2 => "SSLv2", - rustls::ProtocolVersion::SSLv3 => "SSLv3", - rustls::ProtocolVersion::TLSv1_0 => "TLSv1.0", - rustls::ProtocolVersion::TLSv1_1 => "TLSv1.1", - rustls::ProtocolVersion::TLSv1_2 => "TLSv1.2", - rustls::ProtocolVersion::TLSv1_3 => "TLSv1.3", - rustls::ProtocolVersion::DTLSv1_0 => "DTLSv1.0", - rustls::ProtocolVersion::DTLSv1_2 => "DTLSv1.2", - rustls::ProtocolVersion::DTLSv1_3 => "DTLSv1.3", - _ => "unknown", - } - .into(), - match conn.negotiated_cipher_suite() { - Some(rustls::SupportedCipherSuite::Tls13(cs)) => { - cs.common.suite.as_str().unwrap_or("unknown") - } - Some(rustls::SupportedCipherSuite::Tls12(cs)) => { - cs.common.suite.as_str().unwrap_or("unknown") - } - None => "unknown", - } - .into(), - ) - } -} - -impl SessionStream for ProxiedStream { - fn is_tls(&self) -> bool { - self.proxy_header() - .ssl() - .map_or(false, |ssl| ssl.client_ssl()) - } - - fn tls_version_and_cipher(&self) -> (Cow<'static, str>, Cow<'static, str>) { - self.proxy_header() - .ssl() - .map(|ssl| { - ( - ssl.version().unwrap_or("unknown").to_string().into(), - ssl.cipher().unwrap_or("unknown").to_string().into(), - ) - }) - .unwrap_or((Cow::Borrowed("unknown"), Cow::Borrowed("unknown"))) - } -} - -#[derive(Default)] -pub struct NullIo { - pub tx_buf: Vec, -} - -impl AsyncWrite for NullIo { - fn poll_write( - mut self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - buf: &[u8], - ) -> std::task::Poll> { - self.tx_buf.extend_from_slice(buf); - std::task::Poll::Ready(Ok(buf.len())) - } - - fn poll_flush( - self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::task::Poll::Ready(Ok(())) - } - - fn poll_shutdown( - self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - std::task::Poll::Ready(Ok(())) - } -} - -impl AsyncRead for NullIo { - fn poll_read( - self: std::pin::Pin<&mut Self>, - _cx: &mut std::task::Context<'_>, - _buf: &mut tokio::io::ReadBuf<'_>, - ) -> std::task::Poll> { - unreachable!() - } -} - -impl SessionStream for NullIo { - fn is_tls(&self) -> bool { - true - } - - fn tls_version_and_cipher( - &self, - ) -> ( - std::borrow::Cow<'static, str>, - std::borrow::Cow<'static, str>, - ) { - ( - std::borrow::Cow::Borrowed(""), - std::borrow::Cow::Borrowed(""), - ) - } -} diff --git a/crates/utils/src/listener/tls.rs b/crates/utils/src/listener/tls.rs deleted file mode 100644 index 91376231..00000000 --- a/crates/utils/src/listener/tls.rs +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of Stalwart Mail Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::{ - fmt::{self, Formatter}, - path::PathBuf, - sync::Arc, -}; - -use ahash::AHashMap; -use arc_swap::ArcSwap; -use rustls::{ - client::verify_server_name, - server::{ClientHello, ParsedCertificate, ResolvesServerCert}, - sign::CertifiedKey, - version::{TLS12, TLS13}, - Error, SupportedProtocolVersion, -}; -use rustls_pki_types::{DnsName, ServerName}; -use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt}; -use tokio_rustls::{Accept, LazyConfigAcceptor, TlsAcceptor}; - -use crate::{acme::resolver::IsTlsAlpnChallenge, config::tls::build_certified_key}; - -use super::{SessionStream, TcpAcceptor, TcpAcceptorResult}; - -pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; -pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; - -pub struct CertificateResolver { - pub sni: AHashMap>, - pub cert: Arc, -} - -pub struct Certificate { - pub cert: ArcSwap, - pub path: Vec, -} - -impl CertificateResolver { - pub fn add(&mut self, name: &str, ck: Arc) -> Result<(), Error> { - let server_name = { - let checked_name = DnsName::try_from(name) - .map_err(|_| Error::General("Bad DNS name".into())) - .map(|name| name.to_lowercase_owned())?; - ServerName::DnsName(checked_name) - }; - - ck.cert - .load() - .end_entity_cert() - .and_then(ParsedCertificate::try_from) - .and_then(|cert| verify_server_name(&cert, &server_name))?; - - if let ServerName::DnsName(name) = server_name { - self.sni.insert(name.as_ref().to_string(), ck); - } - Ok(()) - } -} - -impl ResolvesServerCert for CertificateResolver { - fn resolve(&self, hello: ClientHello<'_>) -> Option> { - if !self.sni.is_empty() { - if let Some(cert) = hello.server_name().and_then(|name| self.sni.get(name)) { - return cert.cert.load().clone().into(); - } - } - self.cert.cert.load().clone().into() - } -} - -impl TcpAcceptor { - pub async fn accept(&self, stream: IO) -> TcpAcceptorResult - where - IO: SessionStream, - { - match self { - TcpAcceptor::Tls(acceptor) => TcpAcceptorResult::Tls(acceptor.accept(stream)), - TcpAcceptor::Acme { - challenge, - default, - manager, - } => { - if manager.has_order_in_progress() { - match LazyConfigAcceptor::new(Default::default(), stream).await { - Ok(start_handshake) => { - if start_handshake.client_hello().is_tls_alpn_challenge() { - match start_handshake.into_stream(challenge.clone()).await { - Ok(mut tls) => { - tracing::debug!( - context = "acme", - event = "validation", - "Received TLS-ALPN-01 validation request." - ); - let _ = tls.shutdown().await; - } - Err(err) => { - tracing::info!( - context = "acme", - event = "error", - error = ?err, - "TLS-ALPN-01 validation request failed." - ); - } - } - } else { - return TcpAcceptorResult::Tls( - start_handshake.into_stream(default.clone()), - ); - } - } - Err(err) => { - tracing::debug!( - context = "listener", - event = "error", - error = ?err, - "TLS handshake failed." - ); - } - } - - TcpAcceptorResult::Close - } else { - TcpAcceptorResult::Tls(TlsAcceptor::from(default.clone()).accept(stream)) - } - } - TcpAcceptor::Plain => TcpAcceptorResult::Plain(stream), - } - } - - pub fn is_tls(&self) -> bool { - matches!(self, TcpAcceptor::Tls(_) | TcpAcceptor::Acme { .. }) - } -} - -impl TcpAcceptorResult -where - IO: AsyncRead + AsyncWrite + Unpin, -{ - pub fn unwrap_tls(self) -> Accept { - match self { - TcpAcceptorResult::Tls(accept) => accept, - _ => panic!("unwrap_tls called on non-TLS acceptor"), - } - } -} - -impl Certificate { - pub async fn reload(&self) -> crate::config::Result<()> { - let cert = build_certified_key( - tokio::fs::read(&self.path[0]).await.map_err(|err| { - format!( - "Failed to read certificate from path {id:?}: {err}", - id = self.path[0] - ) - })?, - tokio::fs::read(&self.path[1]).await.map_err(|err| { - format!( - "Failed to read private key from path {id:?}: {err}", - id = self.path[1] - ) - })?, - "certificate", - )?; - - self.cert.store(Arc::new(cert)); - - Ok(()) - } -} - -impl std::fmt::Debug for CertificateResolver { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("CertificateResolver") - .field("sni", &self.sni.keys()) - .field("cert", &self.cert.path) - .finish() - } -} diff --git a/tests/Cargo.toml b/tests/Cargo.toml index bd35b8ea..8b887d83 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -26,6 +26,7 @@ jmap_proto = { path = "../crates/jmap-proto" } imap = { path = "../crates/imap", features = ["test_mode"] } imap_proto = { path = "../crates/imap-proto" } smtp = { path = "../crates/smtp", features = ["test_mode", "local_delivery"] } +common = { path = "../crates/common", features = ["test_mode"] } managesieve = { path = "../crates/managesieve", features = ["test_mode"] } smtp-proto = { version = "0.1" } mail-send = { version = "0.4", default-features = false, features = ["cram-md5"] } @@ -53,10 +54,10 @@ ece = "2.2" hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" -base64 = "0.21" +base64 = "0.22" dashmap = "5.4" ahash = { version = "0.8" } -serial_test = "2.0.0" +serial_test = "3.0.0" num_cpus = "1.15.0" async-trait = "0.1.68" chrono = "0.4" diff --git a/tests/src/directory/imap.rs b/tests/src/directory/imap.rs index c79980e0..eadba5ba 100644 --- a/tests/src/directory/imap.rs +++ b/tests/src/directory/imap.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use common::listener::limiter::{ConcurrencyLimiter, InFlight}; use directory::QueryBy; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; @@ -33,8 +34,6 @@ use tokio::{ }; use tokio_rustls::TlsAcceptor; -use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; - use crate::directory::{DirectoryTest, Item, LookupResult}; use super::dummy_tls_acceptor; diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index 6f896dd5..60183ec1 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -27,16 +27,17 @@ pub mod ldap; pub mod smtp; pub mod sql; +use common::config::smtp::session::AddressMapping; use directory::{ - backend::internal::manage::ManageDirectory, core::config::ConfigDirectory, AddressMapping, - Directories, Principal, + backend::internal::manage::ManageDirectory, core::config::ConfigDirectory, Directories, + Principal, }; use mail_send::Credentials; use rustls::ServerConfig; use rustls_pemfile::{certs, pkcs8_private_keys}; use rustls_pki_types::PrivateKeyDer; use std::{borrow::Cow, io::BufReader, path::PathBuf, sync::Arc}; -use store::{config::ConfigStore, LookupStore, Store, Stores}; +use store::{LookupStore, Store, Stores}; use tokio_rustls::TlsAcceptor; use crate::store::TempDir; diff --git a/tests/src/directory/smtp.rs b/tests/src/directory/smtp.rs index 9aa38e5f..f3ead856 100644 --- a/tests/src/directory/smtp.rs +++ b/tests/src/directory/smtp.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use common::listener::limiter::{ConcurrencyLimiter, InFlight}; use directory::{DirectoryError, QueryBy}; use mail_parser::decoders::base64::base64_decode; use mail_send::Credentials; @@ -33,8 +34,6 @@ use tokio::{ }; use tokio_rustls::TlsAcceptor; -use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; - use crate::directory::{DirectoryTest, Item, LookupResult}; use super::dummy_tls_acceptor; diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 11cb18f1..d6b97ab3 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -38,7 +38,8 @@ pub mod thread; use std::{path::PathBuf, sync::Arc, time::Duration}; use ::managesieve::core::ManageSieveSessionManager; -use ::store::config::ConfigStore; +use common::config::server::ServerProtocol; + use ahash::AHashSet; use directory::{backend::internal::manage::ManageDirectory, core::config::ConfigDirectory}; use imap::core::{ImapSessionManager, IMAP}; @@ -50,7 +51,7 @@ use tokio::{ net::TcpStream, sync::{mpsc, watch}, }; -use utils::{config::ServerProtocol, UnwrapFailure}; +use utils::UnwrapFailure; use crate::{add_test_certs, directory::DirectoryStore, store::TempDir}; @@ -278,7 +279,10 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { let mut servers = config.parse_servers().unwrap(); let stores = config.parse_stores().await.failed("Invalid configuration"); let directory = config - .parse_directory(&stores, stores.stores.get(store_id).unwrap().clone()) + .parse_directory( + &stores, + stores.core.storage.datas.get(store_id).unwrap().clone(), + ) .await .unwrap(); @@ -346,11 +350,16 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { .await; if delete_if_exists { - jmap.store.destroy().await; + jmap.core.storage.data.destroy().await; } // Assign Id 0 to admin (required for some tests) - jmap.store.get_or_create_account_id("admin").await.unwrap(); + jmap.core + .storage + .data + .get_or_create_account_id("admin") + .await + .unwrap(); IMAPTest { jmap, diff --git a/tests/src/jmap/auth_acl.rs b/tests/src/jmap/auth_acl.rs index ad2690fd..d7e6163f 100644 --- a/tests/src/jmap/auth_acl.rs +++ b/tests/src/jmap/auth_acl.rs @@ -49,41 +49,57 @@ pub async fn test(params: &mut JMAPTest) { let trash_id = Id::new(TRASH_ID as u64).to_string(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params + .core + .storage .directory .create_test_user_with_email("jane.smith@example.com", "abcde", "Jane Smith") .await; params + .core + .storage .directory .create_test_user_with_email("bill@example.com", "098765", "Bill Foobar") .await; params + .core + .storage .directory .create_test_group_with_email("sales@example.com", "Sales Group") .await; let john_id: Id = server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap() .into(); let jane_id: Id = server - .store + .core + .storage + .data .get_or_create_account_id("jane.smith@example.com") .await .unwrap() .into(); let bill_id: Id = server - .store + .core + .storage + .data .get_or_create_account_id("bill@example.com") .await .unwrap() .into(); let sales_id: Id = server - .store + .core + .storage + .data .get_or_create_account_id("sales@example.com") .await .unwrap() @@ -676,11 +692,13 @@ pub async fn test(params: &mut JMAPTest) { // Add John and Jane to the Sales group for name in ["jdoe@example.com", "jane.smith@example.com"] { params + .core + .storage .directory .add_to_group(name, "sales@example.com") .await; } - server.access_tokens.clear(); + server.inner.access_tokens.clear(); john_client.refresh_session().await.unwrap(); jane_client.refresh_session().await.unwrap(); bill_client.refresh_session().await.unwrap(); @@ -775,10 +793,12 @@ pub async fn test(params: &mut JMAPTest) { // Remove John from the sales group params + .core + .storage .directory .remove_from_group("jdoe@example.com", "sales@example.com") .await; - server.sessions.clear(); + server.inner.sessions.clear(); assert_forbidden( john_client .set_default_account_id(&sales_id.to_string()) diff --git a/tests/src/jmap/auth_limits.rs b/tests/src/jmap/auth_limits.rs index 90274b48..1c95caea 100644 --- a/tests/src/jmap/auth_limits.rs +++ b/tests/src/jmap/auth_limits.rs @@ -23,6 +23,7 @@ use std::{sync::Arc, time::Duration}; +use common::listener::blocked::BLOCKED_IP_KEY; use directory::backend::internal::manage::ManageDirectory; use imap_proto::ResponseType; use jmap::services::housekeeper::Event; @@ -32,7 +33,7 @@ use jmap_client::{ mailbox::{self}, }; use jmap_proto::types::id::Id; -use store::{dispatch::blocked::BLOCKED_IP_KEY, write::now}; +use store::write::now; use crate::{ imap::{ImapConnection, Type}, @@ -47,18 +48,24 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), ) .to_string(); params + .core + .storage .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") .await; @@ -119,7 +126,9 @@ pub async fn test(params: &mut JMAPTest) { // Test fail2ban assert_eq!( server - .store + .core + .storage + .data .config_get(format!("{BLOCKED_IP_KEY}.127.0.0.1")) .await .unwrap(), @@ -139,7 +148,9 @@ pub async fn test(params: &mut JMAPTest) { // Make sure the IP address is blocked assert_eq!( server - .store + .core + .storage + .data .config_get(format!("{BLOCKED_IP_KEY}.127.0.0.1")) .await .unwrap(), @@ -152,7 +163,9 @@ pub async fn test(params: &mut JMAPTest) { // Lift ban server - .store + .core + .storage + .data .config_clear(format!("{BLOCKED_IP_KEY}.127.0.0.1")) .await .unwrap(); diff --git a/tests/src/jmap/auth_oauth.rs b/tests/src/jmap/auth_oauth.rs index 8a69ab51..92677eae 100644 --- a/tests/src/jmap/auth_oauth.rs +++ b/tests/src/jmap/auth_oauth.rs @@ -45,12 +45,16 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let john_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), diff --git a/tests/src/jmap/blob.rs b/tests/src/jmap/blob.rs index faa073fc..7b31f7e7 100644 --- a/tests/src/jmap/blob.rs +++ b/tests/src/jmap/blob.rs @@ -34,18 +34,22 @@ pub async fn test(params: &mut JMAPTest) { println!("Running blob tests..."); let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), ); - server.store.blob_expire_all().await; + server.core.storage.data.blob_expire_all().await; // Blob/set simple test let response = jmap_json_request( @@ -195,7 +199,7 @@ pub async fn test(params: &mut JMAPTest) { ); } - server.store.blob_expire_all().await; + server.core.storage.data.blob_expire_all().await; // Blob/upload Complex Example let response = jmap_json_request( @@ -289,7 +293,7 @@ pub async fn test(params: &mut JMAPTest) { "Pointer {pointer:?} Response: {response:?}", ); } - server.store.blob_expire_all().await; + server.core.storage.data.blob_expire_all().await; // Blob/get Example with Range and Encoding Errors let response = jmap_json_request( @@ -428,7 +432,7 @@ pub async fn test(params: &mut JMAPTest) { "Pointer {pointer:?} Response: {response:?}", ); } - server.store.blob_expire_all().await; + server.core.storage.data.blob_expire_all().await; // Blob/lookup params.client.set_default_account_id(account_id.to_string()); diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 09e7a0a1..0694bf97 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -42,12 +42,16 @@ pub async fn test(params: &mut JMAPTest) { let server = params.server.clone(); let client = &mut params.client; params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 8d69f85a..16367a78 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -42,20 +42,28 @@ pub async fn test(params: &mut JMAPTest) { // Create a domain name and a test account let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params + .core + .storage .directory .create_test_user_with_email("jane@example.com", "abcdef", "Jane Smith") .await; params + .core + .storage .directory .create_test_user_with_email("bill@example.com", "098765", "Bill Foobar") .await; let account_id_1 = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), @@ -63,7 +71,9 @@ pub async fn test(params: &mut JMAPTest) { .to_string(); let account_id_2 = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jane@example.com") .await .unwrap(), @@ -71,27 +81,37 @@ pub async fn test(params: &mut JMAPTest) { .to_string(); let account_id_3 = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("bill@example.com") .await .unwrap(), ) .to_string(); params + .core + .storage .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") .await; // Create a mailing list params + .core + .storage .directory .link_test_address("jdoe@example.com", "members@example.com", "list") .await; params + .core + .storage .directory .link_test_address("jane@example.com", "members@example.com", "list") .await; params + .core + .storage .directory .link_test_address("bill@example.com", "members@example.com", "list") .await; @@ -235,6 +255,8 @@ pub async fn test(params: &mut JMAPTest) { // Removing members from the mailing list and chunked ingest params + .core + .storage .directory .remove_test_alias("jdoe@example.com", "members@example.com") .await; diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs index 28384856..409f2df6 100644 --- a/tests/src/jmap/email_changes.rs +++ b/tests/src/jmap/email_changes.rs @@ -169,7 +169,9 @@ pub async fn test(params: &mut JMAPTest) { } server - .store + .core + .storage + .data .write( BatchBuilder::new() .with_account_id(1) diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 4cc42fdc..9889e0c6 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -63,7 +63,7 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { for mailbox_id in 1545..3010 { batch.create_document(mailbox_id); } - server.store.write(batch.build()).await.unwrap(); + server.core.storage.data.write(batch.build()).await.unwrap(); // Create test messages println!("Inserting JMAP Mail query test messages..."); @@ -79,7 +79,7 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { .delete_document(mailbox_id) .clear(ValueClass::Property(Property::EmailIds.into())); } - server.store.write(batch.build()).await.unwrap(); + server.core.storage.data.write(batch.build()).await.unwrap(); for thread_id in 0..MAX_THREADS { assert!( diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index 9935fc8f..c07b6165 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -149,7 +149,9 @@ pub async fn test(params: &mut JMAPTest) { let id = *id_map.get(from).unwrap(); let new_id = Id::from_parts(thread_id, id.document_id()); server - .store + .core + .storage + .data .write( BatchBuilder::new() .with_account_id(1) @@ -289,7 +291,13 @@ pub async fn test(params: &mut JMAPTest) { { batch.delete_document(thread_id); } - server.store.write(batch.build_batch()).await.unwrap(); + server + .core + .storage + .data + .write(batch.build_batch()) + .await + .unwrap(); assert_is_empty(server).await; } diff --git a/tests/src/jmap/email_submission.rs b/tests/src/jmap/email_submission.rs index 3ae38fc8..b30c0702 100644 --- a/tests/src/jmap/email_submission.rs +++ b/tests/src/jmap/email_submission.rs @@ -94,16 +94,22 @@ pub async fn test(params: &mut JMAPTest) { // Create a test account let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params + .core + .storage .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), diff --git a/tests/src/jmap/event_source.rs b/tests/src/jmap/event_source.rs index 7173a705..ed6fe910 100644 --- a/tests/src/jmap/event_source.rs +++ b/tests/src/jmap/event_source.rs @@ -43,12 +43,16 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 62ea7648..7e730b76 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -24,6 +24,7 @@ use std::{sync::Arc, time::Duration}; use base64::{engine::general_purpose, Engine}; +use common::config::server::ServerProtocol; use directory::core::config::ConfigDirectory; use imap::core::{ImapSessionManager, IMAP}; use jmap::{ @@ -35,9 +36,9 @@ use jmap_client::client::{Client, Credentials}; use jmap_proto::types::id::Id; use reqwest::header; use smtp::core::{SmtpSessionManager, SMTP}; -use store::config::ConfigStore; + use tokio::sync::{mpsc, watch}; -use utils::{config::ServerProtocol, UnwrapFailure}; +use utils::UnwrapFailure; use crate::{add_test_certs, directory::DirectoryStore, store::TempDir}; @@ -386,8 +387,10 @@ pub async fn assert_is_empty(server: Arc) { // Assert is empty server - .store - .assert_is_empty(server.blob_store.clone()) + .core + .storage + .data + .assert_is_empty(server.core.storage.blob.clone()) .await; } @@ -403,7 +406,10 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { let mut servers = config.parse_servers().unwrap(); let stores = config.parse_stores().await.failed("Invalid configuration"); let directory = config - .parse_directory(&stores, stores.stores.get(store_id).unwrap().clone()) + .parse_directory( + &stores, + stores.core.storage.datas.get(store_id).unwrap().clone(), + ) .await .unwrap(); @@ -426,7 +432,12 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { let imap: Arc = IMAP::init(&config) .await .failed("Invalid configuration file"); - jmap.directory.blocked_ips.reload(&config).unwrap(); + jmap.core + .storage + .directory + .blocked_ips + .reload(&config) + .unwrap(); let (shutdown_tx, _) = servers.spawn(|server, shutdown_rx| { match &server.protocol { @@ -446,7 +457,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { // Create tables let directory = DirectoryStore { - store: stores.lookup_stores.get("auth").unwrap().clone(), + store: stores.core.storage.lookups.get("auth").unwrap().clone(), }; directory.create_test_directory().await; directory @@ -454,7 +465,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { .await; if delete_if_exists { - jmap.store.destroy().await; + jmap.core.storage.data.destroy().await; } // Create client diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 99b1b436..3e0e639c 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -30,6 +30,7 @@ use std::{ }; use base64::{engine::general_purpose, Engine}; +use common::listener::SessionData; use directory::backend::internal::manage::ManageDirectory; use ece::EcKeyComponents; use hyper::{body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn, StatusCode}; @@ -47,7 +48,6 @@ use jmap_proto::types::{id::Id, type_state::DataType}; use store::ahash::AHashSet; use tokio::sync::mpsc; -use utils::listener::SessionData; use crate::{ add_test_certs, @@ -84,12 +84,16 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), @@ -285,9 +289,9 @@ struct PushVerification { pub verification_code: String, } -impl utils::listener::SessionManager for SessionManager { +impl common::listener::SessionManager for SessionManager { #[allow(clippy::manual_async_fn)] - fn handle( + fn handle( self, session: SessionData, ) -> impl std::future::Future + Send { @@ -356,10 +360,6 @@ impl utils::listener::SessionManager for SessionManager { fn shutdown(&self) -> impl std::future::Future + Send { async {} } - - fn is_ip_blocked(&self, _: &std::net::IpAddr) -> bool { - false - } } async fn expect_push(event_rx: &mut mpsc::Receiver) -> PushMessage { diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index ade4c2f1..f5c15b7d 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -39,38 +39,50 @@ pub async fn test(params: &mut JMAPTest) { println!("Running quota tests..."); let server = params.server.clone(); params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params + .core + .storage .directory .create_test_user_with_email("robert@example.com", "aabbcc", "Robert Foobar") .await; let other_account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), ); let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("robert@example.com") .await .unwrap(), ); params + .core + .storage .directory .set_test_quota("robert@example.com", 1024) .await; params + .core + .storage .directory .add_to_group("robert@example.com", "jdoe@example.com") .await; // Delete temporary blobs from previous tests - server.store.blob_expire_all().await; + server.core.storage.data.blob_expire_all().await; // Test temporary blob quota (3 files) DISABLE_UPLOAD_QUOTA.store(false, std::sync::atomic::Ordering::Relaxed); @@ -93,7 +105,7 @@ pub async fn test(params: &mut JMAPTest) { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } - server.store.blob_expire_all().await; + server.core.storage.data.blob_expire_all().await; // Test temporary blob quota (50000 bytes) for i in 0..2 { @@ -114,7 +126,7 @@ pub async fn test(params: &mut JMAPTest) { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } - server.store.blob_expire_all().await; + server.core.storage.data.blob_expire_all().await; // Test JMAP Quotas extension let response = jmap_raw_request( diff --git a/tests/src/jmap/sieve_script.rs b/tests/src/jmap/sieve_script.rs index adedeb61..283b4371 100644 --- a/tests/src/jmap/sieve_script.rs +++ b/tests/src/jmap/sieve_script.rs @@ -51,12 +51,16 @@ pub async fn test(params: &mut JMAPTest) { // Create test account params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index 5bacacab..dad78d87 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -42,7 +42,13 @@ const NUM_PASSES: usize = 1; pub async fn test(server: Arc, mut client: Client) { println!("Running concurrency stress tests..."); - server.store.get_or_create_account_id("john").await.unwrap(); + server + .core + .storage + .data + .get_or_create_account_id("john") + .await + .unwrap(); client.set_default_account_id(Id::from(TEST_USER_ID).to_string()); let client = Arc::new(client); email_tests(server.clone(), client.clone()).await; diff --git a/tests/src/jmap/vacation_response.rs b/tests/src/jmap/vacation_response.rs index eb3c1e44..b1c52743 100644 --- a/tests/src/jmap/vacation_response.rs +++ b/tests/src/jmap/vacation_response.rs @@ -45,12 +45,16 @@ pub async fn test(params: &mut JMAPTest) { let server = params.server.clone(); let client = &mut params.client; params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), diff --git a/tests/src/jmap/websocket.rs b/tests/src/jmap/websocket.rs index 80ac2e39..4c29f145 100644 --- a/tests/src/jmap/websocket.rs +++ b/tests/src/jmap/websocket.rs @@ -47,12 +47,16 @@ pub async fn test(params: &mut JMAPTest) { // Authenticate all accounts params + .core + .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; let account_id = Id::from( server - .store + .core + .storage + .data .get_or_create_account_id("jdoe@example.com") .await .unwrap(), diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index 14dd3f06..5f8bb48b 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -23,25 +23,17 @@ use std::{fs, net::IpAddr, path::PathBuf, time::Duration}; -use store::config::ConfigStore; -use tokio::net::TcpSocket; - -use utils::{ +use common::{ config::{ - if_block::{IfBlock, IfThen}, - Config, Listener, Rate, Server, ServerProtocol, + server::{Listener, Server, ServerProtocol}, + smtp::*, }, - expr::{BinaryOperator, Constant, Expression, ExpressionItem, UnaryOperator}, + expr::{functions::ResolveVariable, if_block::*, *}, listener::TcpAcceptor, }; +use tokio::net::TcpSocket; -use smtp::{ - config::{ - map_expr_token, throttle::ConfigThrottle, ConfigContext, Throttle, THROTTLE_AUTH_AS, - THROTTLE_REMOTE_IP, THROTTLE_SENDER_DOMAIN, - }, - core::{eval::*, ResolveVariable}, -}; +use utils::config::{Config, Rate}; use super::add_test_certs; @@ -348,9 +340,6 @@ fn parse_servers() { let expected_servers = vec![ Server { id: "smtp".to_string(), - internal_id: 0, - hostname: "mx.example.org".to_string(), - data: "Stalwart SMTP - hi there!".to_string(), protocol: ServerProtocol::Smtp, listeners: vec![Listener { socket: TcpSocket::new_v4().unwrap(), @@ -367,9 +356,6 @@ fn parse_servers() { }, Server { id: "smtps".to_string(), - internal_id: 1, - hostname: "mx.example.org".to_string(), - data: "Stalwart SMTP - hi there!".to_string(), protocol: ServerProtocol::Smtp, listeners: vec![ Listener { @@ -396,9 +382,6 @@ fn parse_servers() { }, Server { id: "submission".to_string(), - internal_id: 2, - hostname: "submit.example.org".to_string(), - data: "Stalwart SMTP submission at your service".to_string(), protocol: ServerProtocol::Smtp, listeners: vec![Listener { socket: TcpSocket::new_v4().unwrap(), @@ -480,12 +463,10 @@ async fn eval_if() { let servers = vec![ Server { id: "smtp".to_string(), - internal_id: 123, ..Default::default() }, Server { id: "smtps".to_string(), - internal_id: 456, ..Default::default() }, ]; @@ -596,7 +577,7 @@ async fn eval_dynvalue() { } impl ResolveVariable for TestEnvelope { - fn resolve_variable(&self, variable: u32) -> utils::expr::Variable<'_> { + fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_RECIPIENT => self.rcpt.as_str().into(), V_RECIPIENT_DOMAIN => self.rcpt_domain.as_str().into(), diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index b69f584e..e596ad43 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -9,20 +9,22 @@ use std::{ use crate::smtp::session::TestSession; use ahash::AHashMap; +use common::{ + expr::if_block::IfBlock, + scripts::{ + functions::html::{get_attribute, html_attr_tokens, html_img_area, html_to_tokens}, + ScriptModification, + }, +}; use mail_auth::{dmarc::Policy, DkimResult, DmarcResult, IprevResult, SpfResult, MX}; use sieve::runtime::Variable; use smtp::{ - config::{scripts::ConfigSieve, ConfigContext}, core::{Session, SessionAddress, SMTP}, inbound::AuthResult, - scripts::{ - functions::html::{get_attribute, html_attr_tokens, html_img_area, html_to_tokens}, - ScriptModification, ScriptResult, - }, + scripts::ScriptResult, }; -use store::config::ConfigStore; use tokio::runtime::Handle; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; use crate::smtp::{TestConfig, TestSMTP}; @@ -229,9 +231,9 @@ async fn antispam() { let mut ctx = ConfigContext::new(); ctx.stores = config.parse_stores().await.unwrap(); core.sieve = config.parse_sieve(&mut ctx).unwrap(); - core.shared.lookup_stores = ctx.stores.lookup_stores.clone(); - core.shared.scripts = ctx.scripts.clone(); - let config = &mut core.session.config; + core.core.storage.lookups = ctx.stores.lookups.clone(); + core.core.storage.scripts = ctx.scripts.clone(); + let config = &mut core.core.smtp.session; config.rcpt.relay = IfBlock::new(true); // Add mock DNS entries @@ -267,7 +269,7 @@ async fn antispam() { "127.0.0.8", ), ] { - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( domain, vec![ip.parse().unwrap()], Instant::now() + Duration::from_secs(100), @@ -279,7 +281,7 @@ async fn antispam() { "gmail.com", "custom.disposable.org", ] { - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( mx, vec![MX { exchanges: vec!["127.0.0.1".parse().unwrap()], diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index c2b89076..9cae4672 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -21,19 +21,17 @@ * for more details. */ +use common::{config::smtp::session::Mechanism, expr::if_block::IfBlock}; use directory::core::config::ConfigDirectory; use store::Store; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; use crate::smtp::{ inbound::dummy_stores, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }; -use smtp::{ - config::session::Mechanism, - core::{Session, State, SMTP}, -}; +use smtp::core::{Session, State, SMTP}; const DIRECTORY: &str = r#" [storage] @@ -62,14 +60,14 @@ member-of = ["sales", "support"] #[tokio::test] async fn auth() { let mut core = SMTP::test(); - core.shared.directories = Config::new(DIRECTORY) + core.core.storage.directories = Config::new(DIRECTORY) .unwrap() .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; - let config = &mut core.session.config.auth; + let config = &mut core.core.smtp.session.auth; config.require = r#"[{if = "remote_ip = '10.0.0.1'", then = true}, {else = false}]"# @@ -85,7 +83,7 @@ async fn auth() { {else = 0}]"# .parse_if_constant::(); config.must_match_sender = IfBlock::new(true); - core.session.config.extensions.future_release = + core.core.smtp.session.extensions.future_release = r"[{if = '!is_empty(authenticated_as)', then = '1d'}, {else = false}]" .parse_if(); diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 3c3d26d3..48beb418 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -23,9 +23,10 @@ use std::sync::Arc; +use common::expr::if_block::IfBlock; use directory::core::config::ConfigDirectory; use store::Store; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; use crate::smtp::{ inbound::{dummy_stores, TestMessage}, @@ -80,16 +81,16 @@ async fn data() { // Create temp dir for queue let mut qr = core.init_test_queue("smtp_data_test"); - core.shared.directories = Config::new(DIRECTORY) + core.core.storage.directories = Config::new(DIRECTORY) .unwrap() .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.directory = IfBlock::new("local".to_string()); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.data.add_auth_results = r#"[{if = "remote_ip = '10.0.0.3'", then = true}, {else = false}]"# .parse_if(); @@ -103,7 +104,7 @@ async fn data() { {else = 100}]"# .parse_if(); - core.queue.config.quota = r#"[[queue.quota]] + core.core.smtp.queue.quota = r#"[[queue.quota]] match = "sender = 'john@doe.org'" key = ['sender'] messages = 1 @@ -244,8 +245,9 @@ async fn data() { // Make sure store is empty qr.clear_queue(&core).await; - core.shared - .default_data_store - .assert_is_empty(core.shared.default_blob_store.clone()) + core.core + .storage + .data + .assert_is_empty(core.core.storage.blob.clone()) .await; } diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index f79441f5..9e0d11e5 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -26,6 +26,10 @@ use std::{ time::{Duration, Instant}, }; +use common::{ + config::smtp::{auth::VerifyStrategy, report::AggregateFrequency}, + expr::if_block::IfBlock, +}; use directory::core::config::ConfigDirectory; use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, @@ -35,17 +39,14 @@ use mail_auth::{ spf::Spf, }; use store::Store; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; use crate::smtp::{ inbound::{dummy_stores, sign::TextConfigContext, TestMessage, TestReportingEvent}, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; -use smtp::{ - config::{AggregateFrequency, ConfigContext, VerifyStrategy}, - core::{Session, SMTP}, -}; +use smtp::core::{Session, SMTP}; const DIRECTORY: &str = r#" [storage] @@ -65,28 +66,28 @@ email = ["jdoe@example.com"] #[tokio::test] async fn dmarc() { let mut core = SMTP::test(); - core.shared.signers = ConfigContext::new().parse_signatures().signers; + core.core.storage.signers = ConfigContext::new().parse_signatures().signers; // Create temp dir for queue let mut qr = core.init_test_queue("smtp_dmarc_test"); // Add SPF, DKIM and DMARC records - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "mx.example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all ra=spf-failures rr=e:f:s:n").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "foobar.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "ed._domainkey.example.com", DomainKey::parse( concat!( @@ -98,7 +99,7 @@ async fn dmarc() { .unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "default._domainkey.example.com", DomainKey::parse( concat!( @@ -113,12 +114,12 @@ async fn dmarc() { .unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "_report._domainkey.example.com", DomainKeyReport::parse(b"ra=dkim-failures; rp=100; rr=d:o:p:s:u:v:x;").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "_dmarc.example.com", Dmarc::parse( concat!( @@ -134,16 +135,16 @@ async fn dmarc() { // Create report channels let mut rr = core.init_test_report(); - core.shared.directories = Config::new(DIRECTORY) + core.core.storage.directories = Config::new(DIRECTORY) .unwrap() .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.directory = IfBlock::new("local".to_string()); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.data.add_auth_results = IfBlock::new(true); config.data.add_date = IfBlock::new(true); config.data.add_message_id = IfBlock::new(true); @@ -151,7 +152,7 @@ async fn dmarc() { config.data.add_return_path = IfBlock::new(true); config.data.add_received_spf = IfBlock::new(true); - let config = &mut core.report.config; + let config = &mut core.core.smtp.report; config.dkim.send = "\"[1, 1s]\"".parse_if(); config.dmarc.send = config.dkim.send.clone(); config.spf.send = config.dkim.send.clone(); @@ -168,7 +169,7 @@ async fn dmarc() { { else = 'strict' }]"# .parse_if_constant::(); - let config = &mut core.report.config; + let config = &mut core.core.smtp.report; config.spf.sign = "\"['rsa']\"".parse_if(); config.dmarc.sign = "\"['rsa']\"".parse_if(); config.dkim.sign = "\"['rsa']\"".parse_if(); @@ -250,7 +251,7 @@ async fn dmarc() { qr.assert_no_events(); // Unaligned DMARC should be rejected - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "test.net", Spf::parse(b"v=spf1 -all").unwrap(), Instant::now() + Duration::from_secs(5), diff --git a/tests/src/smtp/inbound/ehlo.rs b/tests/src/smtp/inbound/ehlo.rs index 17c422af..d2f719b5 100644 --- a/tests/src/smtp/inbound/ehlo.rs +++ b/tests/src/smtp/inbound/ehlo.rs @@ -23,34 +23,31 @@ use std::time::{Duration, Instant}; +use common::{config::smtp::auth::VerifyStrategy, expr::if_block::IfBlock}; use mail_auth::{common::parse::TxtRecordParser, spf::Spf, SpfResult}; use smtp_proto::MtPriority; -use utils::config::if_block::IfBlock; use crate::smtp::{ session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }; -use smtp::{ - config::VerifyStrategy, - core::{Session, SMTP}, -}; +use smtp::core::{Session, SMTP}; #[tokio::test] async fn ehlo() { let mut core = SMTP::test(); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "mx1.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "mx2.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.data.max_message_size = r#"[{if = "remote_ip = '10.0.0.1'", then = 1024}, {else = 2048}]"# .parse_if(); diff --git a/tests/src/smtp/inbound/limits.rs b/tests/src/smtp/inbound/limits.rs index b0b9b0ed..37273827 100644 --- a/tests/src/smtp/inbound/limits.rs +++ b/tests/src/smtp/inbound/limits.rs @@ -34,7 +34,7 @@ use smtp::core::{Session, SMTP}; #[tokio::test] async fn limits() { let mut core = SMTP::test(); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.transfer_limit = r#"[{if = "remote_ip = '10.0.0.1'", then = 10}, {else = 1024}]"# .parse_if(); diff --git a/tests/src/smtp/inbound/mail.rs b/tests/src/smtp/inbound/mail.rs index f662d121..0dcb0ab8 100644 --- a/tests/src/smtp/inbound/mail.rs +++ b/tests/src/smtp/inbound/mail.rs @@ -26,49 +26,46 @@ use std::{ time::{Duration, Instant, SystemTime}, }; +use common::{config::smtp::auth::VerifyStrategy, expr::if_block::IfBlock}; use mail_auth::{common::parse::TxtRecordParser, spf::Spf, IprevResult, SpfResult}; use smtp_proto::{MtPriority, MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; -use utils::config::if_block::IfBlock; use crate::smtp::{ session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }; -use smtp::{ - config::VerifyStrategy, - core::{Session, SMTP}, -}; +use smtp::core::{Session, SMTP}; #[tokio::test] async fn mail() { let mut core = SMTP::test(); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "mx1.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.ptr_add( + core.core.smtp.resolvers.dns.ptr_add( "10.0.0.1".parse().unwrap(), vec!["mx1.foobar.org.".to_string()], Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx1.foobar.org.", vec!["10.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.ptr_add( + core.core.smtp.resolvers.dns.ptr_add( "10.0.0.2".parse().unwrap(), vec!["mx2.foobar.org.".to_string()], Instant::now() + Duration::from_secs(5), ); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.ehlo.require = IfBlock::new(true); core.mail_auth.spf.verify_ehlo = IfBlock::new(VerifyStrategy::Relaxed); core.mail_auth.spf.verify_mail_from = r#"[{if = "remote_ip = '10.0.0.2'", then = 'strict'}, @@ -185,7 +182,7 @@ async fn mail() { .unwrap(); session.response().assert_code("550 5.7.25"); session.data.iprev = None; - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx2.foobar.org.", vec!["10.0.0.2".parse().unwrap()], Instant::now() + Duration::from_secs(5), @@ -197,7 +194,7 @@ async fn mail() { .await .unwrap(); session.response().assert_code("550 5.7.23"); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index 3f3dab82..d10573e1 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -23,15 +23,18 @@ use std::{fs, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; +use common::{ + config::smtp::session::{Milter, MilterVersion}, + expr::if_block::IfBlock, +}; use mail_auth::AuthenticatedMessage; use mail_parser::MessageParser; use serde::Deserialize; use smtp::{ - config::Milter, core::{Session, SessionData, SMTP}, inbound::milter::{ receiver::{FrameResult, Receiver}, - Action, Command, Macros, MilterClient, Modification, Options, Response, Version, + Action, Command, Macros, MilterClient, Modification, Options, Response, }, }; use tokio::{ @@ -39,7 +42,6 @@ use tokio::{ net::{TcpListener, TcpStream}, sync::watch, }; -use utils::config::if_block::IfBlock; use crate::smtp::{ inbound::TestMessage, @@ -69,7 +71,7 @@ async fn milter_session() { tokio::time::sleep(Duration::from_millis(100)).await; let mut core = SMTP::test(); let mut qr = core.init_test_queue("smtp_milter_test"); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.rcpt.relay = IfBlock::new(true); config.data.milters = r#"[[session.data.milter]] hostname = "127.0.0.1" @@ -391,7 +393,7 @@ async fn milter_client_test() { tls_allow_invalid_certs: false, tempfail_on_error: false, max_frame_len: 5000000, - protocol_version: Version::V6, + protocol_version: MilterVersion::V6, flags_actions: None, flags_protocol: None, }, diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs index 0c6843f9..e422078a 100644 --- a/tests/src/smtp/inbound/mod.rs +++ b/tests/src/smtp/inbound/mod.rs @@ -384,7 +384,7 @@ pub fn dummy_stores() -> Stores { let store = Store::default(); stores.stores.insert("dummy".to_string(), store.clone()); stores - .lookup_stores + .lookups .insert("dummy".to_string(), store.clone().into()); stores .fts_stores diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index ab0f940b..dfc6ea71 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -23,10 +23,11 @@ use std::time::Duration; +use common::expr::if_block::IfBlock; use directory::core::config::ConfigDirectory; use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; use store::Store; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; use crate::smtp::{ inbound::dummy_stores, @@ -72,14 +73,14 @@ email = "mike@foobar.org" async fn rcpt() { let mut core = SMTP::test(); - let config_ext = &mut core.session.config.extensions; - core.shared.directories = Config::new(DIRECTORY) + let config_ext = &mut core.core.smtp.session.extensions; + core.core.storage.directories = Config::new(DIRECTORY) .unwrap() .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.directory = IfBlock::new("local".to_string()); config.max_recipients = r#"[{if = "remote_ip = '10.0.0.1'", then = 3}, {else = 5}]"# @@ -96,7 +97,7 @@ async fn rcpt() { config.errors_wait = r#"[{if = "remote_ip = '10.0.0.1'", then = '5ms'}, {else = '1s'}]"# .parse_if(); - core.session.config.throttle.rcpt_to = r#"[[throttle]] + core.core.smtp.session.throttle.rcpt_to = r#"[[throttle]] match = "remote_ip = '10.0.0.1'" key = 'sender' rate = '2/1s' diff --git a/tests/src/smtp/inbound/rewrite.rs b/tests/src/smtp/inbound/rewrite.rs index ca149d51..2c878a25 100644 --- a/tests/src/smtp/inbound/rewrite.rs +++ b/tests/src/smtp/inbound/rewrite.rs @@ -26,13 +26,11 @@ use crate::smtp::{ session::TestSession, TestConfig, }; +use common::{config::smtp::*, expr::if_block::IfBlock}; use directory::core::config::ConfigDirectory; -use smtp::{ - config::{map_expr_token, scripts::ConfigSieve, ConfigContext}, - core::{eval::*, Session, SMTP}, -}; +use smtp::core::{Session, SMTP}; use store::Store; -use utils::config::{if_block::IfBlock, utils::NoConstants, Config}; +use utils::config::Config; const CONFIG: &str = r#" [storage] @@ -107,8 +105,8 @@ async fn address_rewrite() { .await .unwrap(); core.sieve = settings.parse_sieve(&mut ctx).unwrap(); - core.shared.scripts = ctx.scripts; - let config = &mut core.session.config; + core.core.storage.scripts = ctx.scripts; + let config = &mut core.core.smtp.session; config.mail.script = settings .parse_if_block("session.mail.script", |name| { map_expr_token::(name, available_keys) diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 46f9f92c..a556c727 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -29,15 +29,15 @@ use crate::smtp::{ session::{TestSession, VerifyResponse}, TestConfig, TestSMTP, }; +use common::{config::smtp::V_REMOTE_IP, expr::if_block::IfBlock}; use directory::core::config::ConfigDirectory; use smtp::{ - config::{scripts::ConfigSieve, session::ConfigSession, ConfigContext}, - core::{eval::V_REMOTE_IP, Session, SMTP}, + core::{Session, SMTP}, scripts::ScriptResult, }; -use store::{config::ConfigStore, Store}; +use store::Store; use tokio::runtime::Handle; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; const CONFIG: &str = r#" [storage] @@ -136,17 +136,17 @@ async fn sieve_scripts() { ) .unwrap(); ctx.stores = config.parse_stores().await.unwrap(); - core.shared.lookup_stores = ctx.stores.lookup_stores.clone(); - core.shared.directories = config + core.core.storage.lookups = ctx.stores.lookups.clone(); + core.core.storage.directories = config .parse_directory(&ctx.stores, Store::default()) .await .unwrap() .directories; let pipes = config.parse_pipes(&[V_REMOTE_IP]).unwrap(); core.sieve = config.parse_sieve(&mut ctx).unwrap(); - core.shared.signers = ctx.signers; - core.shared.scripts = ctx.scripts.clone(); - let config = &mut core.session.config; + core.core.storage.signers = ctx.signers; + core.core.storage.scripts = ctx.scripts.clone(); + let config = &mut core.core.smtp.session; config.connect.script = IfBlock::new("stage_connect".to_string()); config.ehlo.script = IfBlock::new("stage_ehlo".to_string()); config.mail.script = IfBlock::new("stage_mail".to_string()); diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 7d6a2de4..4c3d1941 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -23,23 +23,21 @@ use std::time::{Duration, Instant}; +use common::{config::smtp::auth::VerifyStrategy, expr::if_block::IfBlock}; use directory::core::config::ConfigDirectory; use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, spf::Spf, }; use store::Store; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; use crate::smtp::{ inbound::{dummy_stores, TestMessage}, session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; -use smtp::{ - config::{auth::ConfigAuth, ConfigContext, VerifyStrategy}, - core::{Session, SMTP}, -}; +use smtp::core::{Session, SMTP}; const SIGNATURES: &str = " [signature.rsa] @@ -115,17 +113,17 @@ async fn sign_and_seal() { let mut qr = core.init_test_queue("smtp_sign_test"); // Add SPF, DKIM and DMARC records - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "mx.example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "ed._domainkey.scamorza.org", DomainKey::parse( concat!( @@ -137,7 +135,7 @@ async fn sign_and_seal() { .unwrap(), Instant::now() + Duration::from_secs(5), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "rsa._domainkey.manchego.org", DomainKey::parse( concat!( @@ -153,16 +151,16 @@ async fn sign_and_seal() { Instant::now() + Duration::from_secs(5), ); - core.shared.directories = Config::new(DIRECTORY) + core.core.storage.directories = Config::new(DIRECTORY) .unwrap() .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.directory = IfBlock::new("local".to_string()); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.data.add_auth_results = IfBlock::new(true); config.data.add_date = IfBlock::new(true); config.data.add_message_id = IfBlock::new(true); @@ -172,8 +170,8 @@ async fn sign_and_seal() { let config = &mut core.mail_auth; let ctx = ConfigContext::new().parse_signatures(); - core.shared.signers = ctx.signers; - core.shared.sealers = ctx.sealers; + core.core.storage.signers = ctx.signers; + core.core.storage.sealers = ctx.sealers; config.spf.verify_ehlo = IfBlock::new(VerifyStrategy::Relaxed); config.spf.verify_mail_from = config.spf.verify_ehlo.clone(); config.dkim.verify = config.spf.verify_ehlo.clone(); diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs index 472fb4d6..becc09a6 100644 --- a/tests/src/smtp/inbound/throttle.rs +++ b/tests/src/smtp/inbound/throttle.rs @@ -39,7 +39,7 @@ async fn throttle_inbound() { let mut core = SMTP::test(); let _qr = core.init_test_queue("smtp_inbound_throttle"); - let config = &mut core.session.config; + let config = &mut core.core.smtp.session; config.throttle.connect = r#"[[throttle]] match = "remote_ip = '10.0.0.1'" key = 'remote_ip' diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 4147af95..98bb0d92 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -21,9 +21,10 @@ * for more details. */ +use common::expr::if_block::IfBlock; use directory::core::config::ConfigDirectory; use store::Store; -use utils::config::{if_block::IfBlock, Config}; +use utils::config::Config; use crate::smtp::{ inbound::dummy_stores, @@ -66,16 +67,16 @@ email-list = ["sales@foobar.org"] async fn vrfy_expn() { let mut core = SMTP::test(); - core.shared.directories = Config::new(DIRECTORY) + core.core.storage.directories = Config::new(DIRECTORY) .unwrap() .parse_directory(&dummy_stores(), Store::default()) .await .unwrap() .directories; - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.directory = IfBlock::new("local".to_string()); - let config = &mut core.session.config.extensions; + let config = &mut core.core.smtp.session.extensions; config.vrfy = r#"[{if = "remote_ip = '10.0.0.1'", then = true}, {else = false}]"# .parse_if(); diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index 95ebb435..d88e5096 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -23,14 +23,15 @@ use std::time::{Duration, Instant}; +use common::{ + config::smtp::{session::Mechanism, *}, + expr::{if_block::IfBlock, Expression}, +}; use directory::core::config::ConfigDirectory; use mail_auth::MX; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; -use store::{config::ConfigStore, Store}; -use utils::{ - config::{if_block::IfBlock, Config}, - expr::Expression, -}; +use store::Store; +use utils::config::Config; use crate::{ directory::DirectoryStore, @@ -41,8 +42,7 @@ use crate::{ store::TempDir, }; use smtp::{ - config::{map_expr_token, session::Mechanism, ConfigContext}, - core::{eval::*, Session, SMTP}, + core::{Session, SMTP}, queue::RecipientDomain, }; @@ -96,13 +96,13 @@ async fn lookup_sql() { let mut ctx = ConfigContext::new(); let config = Config::new(&config_file).unwrap(); ctx.stores = config.parse_stores().await.unwrap(); - core.shared.lookup_stores = ctx.stores.lookup_stores.clone(); - core.shared.directories = config + core.core.storage.lookups = ctx.stores.lookups.clone(); + core.core.storage.directories = config .parse_directory(&ctx.stores, Store::default()) .await .unwrap() .directories; - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "test.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -113,7 +113,7 @@ async fn lookup_sql() { // Obtain directory handle let handle = DirectoryStore { - store: ctx.stores.lookup_stores.get("sql").unwrap().clone(), + store: ctx.stores.lookups.get("sql").unwrap().clone(), }; // Create tables @@ -202,7 +202,8 @@ async fn lookup_sql() { }) .unwrap(); assert_eq!( - core.eval_expr::(&e, &RecipientDomain::new("test.org"), "text") + core.core + .eval_expr::(&e, &RecipientDomain::new("test.org"), "text") .await .unwrap(), expected, @@ -212,19 +213,19 @@ async fn lookup_sql() { } // Enable AUTH - let config = &mut core.session.config.auth; + let config = &mut core.core.smtp.session.auth; config.directory = "\"'sql'\"".parse_if(); config.mechanisms = IfBlock::new(Mechanism::from(AUTH_PLAIN | AUTH_LOGIN)); config.errors_wait = IfBlock::new(Duration::from_millis(5)); // Enable VRFY/EXPN/RCPT - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.directory = "\"'sql'\"".parse_if(); config.relay = IfBlock::new(false); config.errors_wait = IfBlock::new(Duration::from_millis(5)); // Enable REQUIRETLS based on SQL lookup - core.session.config.extensions.requiretls = + core.core.smtp.session.extensions.requiretls = r#"[{if = "key_exists('sql/is_ip_allowed', remote_ip)", then = true}, {else = false}]"# .parse_if(); diff --git a/tests/src/smtp/lookup/utils.rs b/tests/src/smtp/lookup/utils.rs index e73249d8..81648dbb 100644 --- a/tests/src/smtp/lookup/utils.rs +++ b/tests/src/smtp/lookup/utils.rs @@ -23,19 +23,18 @@ use std::time::{Duration, Instant}; +use common::{ + config::smtp::{ + report::AggregateFrequency, + resolver::{Mode, MxPattern, Policy}, + }, + expr::if_block::IfBlock, +}; use mail_auth::{IpLookupStrategy, MX}; use ::smtp::{core::SMTP, outbound::NextHop}; use mail_parser::DateTime; -use smtp::{ - config::AggregateFrequency, - outbound::{ - lookup::ToNextHop, - mta_sts::{Mode, MxPattern, Policy}, - }, - queue::RecipientDomain, -}; -use utils::config::if_block::IfBlock; +use smtp::{outbound::lookup::ToNextHop, queue::RecipientDomain}; use crate::smtp::{ParseTestConfig, TestConfig}; @@ -54,7 +53,7 @@ async fn lookup_ip() { "10.0.0.4".parse().unwrap(), ]; let mut core = SMTP::test(); - core.queue.config.source_ip.ipv4 = format!( + core.core.smtp.queue.source_ip.ipv4 = format!( "\"[{}]\"", ipv4.iter() .map(|ip| format!("'{}'", ip)) @@ -63,7 +62,7 @@ async fn lookup_ip() { ) .as_str() .parse_if(); - core.queue.config.source_ip.ipv6 = format!( + core.core.smtp.queue.source_ip.ipv6 = format!( "\"[{}]\"", ipv6.iter() .map(|ip| format!("'{}'", ip)) @@ -72,7 +71,7 @@ async fn lookup_ip() { ) .as_str() .parse_if(); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec![ "172.168.0.100".parse().unwrap(), @@ -80,14 +79,14 @@ async fn lookup_ip() { ], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv6_add( + core.core.smtp.resolvers.dns.ipv6_add( "mx.foobar.org", vec!["e:f::a".parse().unwrap(), "e:f::b".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); // Ipv4 strategy - core.queue.config.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv4thenIpv6); + core.core.smtp.queue.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv4thenIpv6); let resolve_result = core .resolve_host( &NextHop::MX("mx.foobar.org"), @@ -105,7 +104,7 @@ async fn lookup_ip() { .contains(&"172.168.0.100".parse().unwrap())); // Ipv6 strategy - core.queue.config.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv6thenIpv4); + core.core.smtp.queue.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv6thenIpv4); let resolve_result = core .resolve_host( &NextHop::MX("mx.foobar.org"), diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index 77bb319b..1bc8ae73 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -27,12 +27,13 @@ use std::{ }; use ahash::{AHashMap, HashMap, HashSet}; +use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; use directory::core::config::ConfigDirectory; use mail_auth::MX; use mail_parser::DateTime; use reqwest::{header::AUTHORIZATION, Method, StatusCode}; use store::Store; -use utils::config::{if_block::IfBlock, Config, ServerProtocol}; +use utils::config::Config; use crate::smtp::{ inbound::dummy_stores, management::send_manage_request, outbound::start_test_server, @@ -78,14 +79,14 @@ async fn manage_queue() { // Start remote test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_manage_queue_remote"); let remote_core = Arc::new(core); let _rx_remote = start_test_server(remote_core.clone(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx1.foobar.org".to_string()], @@ -94,7 +95,7 @@ async fn manage_queue() { Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx1.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -106,14 +107,14 @@ async fn manage_queue() { .parse_directory(&dummy_stores(), Store::default()) .await .unwrap(); - core.shared.default_directory = directory.directories.get("local").unwrap().clone(); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.rcpt.max_recipients = IfBlock::new(100); - core.session.config.extensions.future_release = IfBlock::new(Duration::from_secs(86400)); - core.session.config.extensions.dsn = IfBlock::new(true); - core.queue.config.retry = IfBlock::new(Duration::from_secs(1000)); - core.queue.config.notify = IfBlock::new(Duration::from_secs(2000)); - core.queue.config.expire = IfBlock::new(Duration::from_secs(3000)); + core.core.storage.directory = directory.directories.get("local").unwrap().clone(); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.max_recipients = IfBlock::new(100); + core.core.smtp.session.extensions.future_release = IfBlock::new(Duration::from_secs(86400)); + core.core.smtp.session.extensions.dsn = IfBlock::new(true); + core.core.smtp.queue.retry = IfBlock::new(Duration::from_secs(1000)); + core.core.smtp.queue.notify = IfBlock::new(Duration::from_secs(2000)); + core.core.smtp.queue.expire = IfBlock::new(Duration::from_secs(3000)); let local_qr = core.init_test_queue("smtp_manage_queue_local"); let core = Arc::new(core); local_qr.queue_rx.spawn(core.clone()); diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index 27ddac92..0593fb64 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -24,6 +24,10 @@ use std::sync::Arc; use ahash::{AHashMap, HashSet}; +use common::{ + config::{server::ServerProtocol, smtp::report::AggregateFrequency}, + expr::if_block::IfBlock, +}; use directory::core::config::ConfigDirectory; use mail_auth::{ common::parse::TxtRecordParser, @@ -37,7 +41,7 @@ use mail_auth::{ use reqwest::Method; use store::Store; use tokio::sync::mpsc; -use utils::config::{if_block::IfBlock, Config, ServerProtocol}; +use utils::config::Config; use crate::smtp::{ inbound::dummy_stores, @@ -46,7 +50,6 @@ use crate::smtp::{ TestConfig, }; use smtp::{ - config::AggregateFrequency, core::{management::Report, SMTP}, reporting::{scheduler::SpawnReport, DmarcEvent, TlsEvent}, }; @@ -79,7 +82,7 @@ async fn manage_reports() { // Start reporting service let mut core = SMTP::test(); - let config = &mut core.report.config; + let config = &mut core.core.smtp.report; config.dmarc_aggregate.max_size = IfBlock::new(1024); config.tls.max_size = IfBlock::new(1024); let directory = Config::new(DIRECTORY) @@ -87,7 +90,7 @@ async fn manage_reports() { .parse_directory(&dummy_stores(), Store::default()) .await .unwrap(); - core.shared.default_directory = directory.directories.get("local").unwrap().clone(); + core.core.storage.directory = directory.directories.get("local").unwrap().clone(); let (report_tx, report_rx) = mpsc::channel(1024); core.report.tx = report_tx; let core = Arc::new(core); diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 1e353040..8b3b1f94 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -23,8 +23,9 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; +use common::{config::smtp::*, expr::if_block::IfBlock}; use dashmap::DashMap; -use directory::{AddressMapping, Directory, DirectoryInner}; +use directory::{Directory, DirectoryInner}; use mail_auth::{ common::lru::{DnsCache, LruCache}, hickory_resolver::config::{ResolverConfig, ResolverOpts}, @@ -33,34 +34,10 @@ use mail_auth::{ use mail_send::smtp::tls::build_tls_connector; use sieve::Runtime; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; -use store::{ - backend::sqlite::SqliteStore, dispatch::blocked::BlockedIps, BlobStore, LookupStore, Store, -}; +use store::{backend::sqlite::SqliteStore, BlobStore, LookupStore, Store}; use tokio::sync::mpsc; -use smtp::{ - config::{ - map_expr_token, - queue::ConfigQueue, - scripts::SieveContext, - session::{ConfigSession, Mechanism}, - throttle::ConfigThrottle, - AggregateReport, ArcAuthConfig, Auth, Connect, Data, DkimAuthConfig, DmarcAuthConfig, Dsn, - Ehlo, Extensions, IpRevAuthConfig, Mail, MailAuthConfig, Milter, QueueConfig, - QueueOutboundSourceIp, QueueOutboundTimeout, QueueOutboundTls, QueueQuotas, QueueThrottle, - Rcpt, Report, ReportAnalysis, ReportConfig, SessionConfig, SessionThrottle, SpfAuthConfig, - Throttle, VerifyStrategy, - }, - core::{ - eval::*, throttle::ThrottleKeyHasherBuilder, QueueCore, ReportCore, Resolvers, SessionCore, - Shared, SieveCore, TlsConnectors, SMTP, - }, - outbound::dane::DnssecResolver, -}; -use utils::{ - config::{if_block::IfBlock, utils::ConstantValue, Config}, - snowflake::SnowflakeIdGenerator, -}; +use utils::{config::Config, snowflake::SnowflakeIdGenerator}; pub mod config; pub mod inbound; @@ -196,18 +173,18 @@ impl TestConfig for SMTP { signers: Default::default(), sealers: Default::default(), directories: Default::default(), - lookup_stores: Default::default(), + lookups: Default::default(), relay_hosts: Default::default(), - default_directory: Arc::new(Directory { + directory: Arc::new(Directory { store: DirectoryInner::Internal(store.clone()), catch_all: AddressMapping::Disable, subaddressing: AddressMapping::Disable, cache: None, blocked_ips: Arc::new(BlockedIps::new(store.clone().into())), }), - default_lookup_store: LookupStore::Store(store.clone()), - default_blob_store: store.clone().into(), - default_data_store: store, + lookup: LookupStore::Store(store.clone()), + blob: store.clone().into(), + data: store, }, } } @@ -525,11 +502,11 @@ impl TestSMTP for SMTP { Config::new(&QUEUE_STORE_CONFIG.replace("{TMP}", _temp_dir.temp_dir.to_str().unwrap())) .unwrap(); let store = Store::SQLite(SqliteStore::open(&config, "store.sqlite").unwrap().into()); - self.shared.default_data_store = store.clone(); - self.shared.default_blob_store = store.clone().into(); - self.shared.default_lookup_store = store.clone().into(); + self.core.storage.data = store.clone(); + self.core.storage.blob = store.clone().into(); + self.core.storage.lookup = store.clone().into(); let (queue_tx, queue_rx) = mpsc::channel(128); - self.queue.tx = queue_tx; + self.inner.queue_tx = queue_tx; QueueReceiver { blob_store: store.clone().into(), @@ -541,7 +518,7 @@ impl TestSMTP for SMTP { fn init_test_report(&mut self) -> ReportReceiver { let (report_tx, report_rx) = mpsc::channel(128); - self.report.tx = report_tx; + self.inner.report_tx = report_tx; ReportReceiver { report_rx } } } diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs index e0763923..4d2b7f2b 100644 --- a/tests/src/smtp/outbound/dane.rs +++ b/tests/src/smtp/outbound/dane.rs @@ -31,6 +31,17 @@ use std::{ time::{Duration, Instant}, }; +use common::{ + config::{ + server::ServerProtocol, + smtp::{ + queue::RequireOptional, + report::AggregateFrequency, + resolver::{DnsRecordCache, DnssecResolver, Resolvers, Tlsa, TlsaEntry}, + }, + }, + expr::if_block::IfBlock, +}; use mail_auth::{ common::{ lru::{DnsCache, LruCache}, @@ -45,7 +56,7 @@ use mail_auth::{ Resolver, MX, }; use rustls_pki_types::CertificateDer; -use utils::config::{if_block::IfBlock, ServerProtocol}; +use utils::suffixlist::PublicSuffix; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, @@ -54,9 +65,7 @@ use crate::smtp::{ TestConfig, TestSMTP, }; use smtp::{ - config::{AggregateFrequency, RequireOptional}, - core::{Resolvers, Session, SMTP}, - outbound::dane::{DnssecResolver, Tlsa, TlsaEntry}, + core::{Session, SMTP}, queue::{Error, ErrorDetails, Status}, reporting::PolicyType, }; @@ -73,13 +82,13 @@ async fn dane_verify() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_dane_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -87,12 +96,12 @@ async fn dane_verify() { }], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "_smtp._tls.foobar.org", TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), Instant::now() + Duration::from_secs(10), @@ -101,9 +110,9 @@ async fn dane_verify() { // Fail on missing TLSA record let mut local_qr = core.init_test_queue("smtp_dane_local"); let mut rr = core.init_test_report(); - core.session.config.rcpt.relay = IfBlock::new(true); - core.queue.config.tls.dane = IfBlock::new(RequireOptional::Require); - core.report.config.tls.send = IfBlock::new(AggregateFrequency::Weekly); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.queue.tls.dane = IfBlock::new(RequireOptional::Require); + core.core.smtp.report.tls.send = IfBlock::new(AggregateFrequency::Weekly); let core = Arc::new(core); let mut session = Session::test(core.clone()); @@ -156,7 +165,7 @@ async fn dane_verify() { has_end_entities: true, has_intermediates: false, }); - core.resolvers.tlsa_add( + core.core.smtp.resolvers.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), @@ -202,7 +211,7 @@ async fn dane_verify() { has_end_entities: true, has_intermediates: false, }); - core.resolvers.tlsa_add( + core.core.smtp.resolvers.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), @@ -242,10 +251,11 @@ async fn dane_test() { dnssec: DnssecResolver { resolver: AsyncResolver::tokio(conf, opts), }, - cache: smtp::core::DnsCache { + cache: DnsRecordCache { tlsa: LruCache::with_capacity(10), mta_sts: LruCache::with_capacity(10), }, + psl: PublicSuffix::default(), }; // Add dns entries diff --git a/tests/src/smtp/outbound/extensions.rs b/tests/src/smtp/outbound/extensions.rs index bfe2b286..e08fcb6f 100644 --- a/tests/src/smtp/outbound/extensions.rs +++ b/tests/src/smtp/outbound/extensions.rs @@ -26,9 +26,9 @@ use std::{ time::{Duration, Instant}, }; +use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; use mail_auth::MX; use smtp_proto::{MAIL_REQUIRETLS, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_NEVER}; -use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, @@ -50,16 +50,16 @@ async fn extensions() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.data.max_message_size = IfBlock::new(1500); - core.session.config.extensions.dsn = IfBlock::new(true); - core.session.config.extensions.requiretls = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.data.max_message_size = IfBlock::new(1500); + core.core.smtp.session.extensions.dsn = IfBlock::new(true); + core.core.smtp.session.extensions.requiretls = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_ext_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -67,7 +67,7 @@ async fn extensions() { }], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -75,8 +75,8 @@ async fn extensions() { // Successful delivery with DSN let mut local_qr = core.init_test_queue("smtp_ext_local"); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.extensions.dsn = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.extensions.dsn = IfBlock::new(true); let core = Arc::new(core); //let mut queue = Queue::default(); let mut session = Session::test(core.clone()); diff --git a/tests/src/smtp/outbound/ip_lookup.rs b/tests/src/smtp/outbound/ip_lookup.rs index c2f4c2e2..504d1b5c 100644 --- a/tests/src/smtp/outbound/ip_lookup.rs +++ b/tests/src/smtp/outbound/ip_lookup.rs @@ -26,8 +26,8 @@ use std::{ time::{Duration, Instant}, }; +use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; use mail_auth::{IpLookupStrategy, MX}; -use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{outbound::start_test_server, session::TestSession, TestConfig, TestSMTP}; use smtp::core::{Session, SMTP}; @@ -44,7 +44,7 @@ async fn ip_lookup_strategy() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_iplookup_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); @@ -52,8 +52,8 @@ async fn ip_lookup_strategy() { //println!("-> Strategy: {:?}", strategy); // Add mock DNS entries let mut core = SMTP::test(); - core.queue.config.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv6thenIpv4); - core.resolvers.dns.mx_add( + core.core.smtp.queue.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv6thenIpv4); + core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -62,13 +62,13 @@ async fn ip_lookup_strategy() { Instant::now() + Duration::from_secs(10), ); if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); } - core.resolvers.dns.ipv6_add( + core.core.smtp.resolvers.dns.ipv6_add( "mx.foobar.org", vec!["::1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -76,7 +76,7 @@ async fn ip_lookup_strategy() { // Retry on failed STARTTLS let mut local_qr = core.init_test_queue("smtp_iplookup_local"); - core.session.config.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); let core = Arc::new(core); let mut session = Session::test(core.clone()); diff --git a/tests/src/smtp/outbound/lmtp.rs b/tests/src/smtp/outbound/lmtp.rs index ecc76204..715c7d7c 100644 --- a/tests/src/smtp/outbound/lmtp.rs +++ b/tests/src/smtp/outbound/lmtp.rs @@ -32,13 +32,13 @@ use crate::smtp::{ session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; +use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; use smtp::{ - config::shared::ConfigShared, core::{Session, SMTP}, queue::{DeliveryAttempt, Event}, }; use store::write::now; -use utils::config::{if_block::IfBlock, Config, ServerProtocol}; +use utils::config::Config; const REMOTE: &str = " [remote.lmtp] @@ -64,14 +64,14 @@ async fn lmtp_delivery() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.extensions.dsn = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.extensions.dsn = IfBlock::new(true); let mut remote_qr = core.init_test_queue("lmtp_delivery_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Lmtp]); // Add mock DNS entries let mut core = SMTP::test(); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "lmtp.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -79,17 +79,17 @@ async fn lmtp_delivery() { // Multiple delivery attempts let mut local_qr = core.init_test_queue("lmtp_delivery_local"); - core.shared.relay_hosts.insert( + core.core.storage.relay_hosts.insert( "lmtp".to_string(), Config::new(REMOTE).unwrap().parse_host("lmtp").unwrap(), ); - core.queue.config.next_hop = r#"[{if = "rcpt_domain = 'foobar.org'", then = "'lmtp'"}, + core.core.smtp.queue.next_hop = r#"[{if = "rcpt_domain = 'foobar.org'", then = "'lmtp'"}, {else = false}]"# .parse_if(); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.rcpt.max_recipients = IfBlock::new(100); - core.session.config.extensions.dsn = IfBlock::new(true); - let config = &mut core.queue.config; + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.max_recipients = IfBlock::new(100); + core.core.smtp.session.extensions.dsn = IfBlock::new(true); + let config = &mut core.core.smtp.queue; config.retry = IfBlock::new(Duration::from_secs(1)); config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, {else = [1s]}]"# diff --git a/tests/src/smtp/outbound/mod.rs b/tests/src/smtp/outbound/mod.rs index 10962ffa..5e771a50 100644 --- a/tests/src/smtp/outbound/mod.rs +++ b/tests/src/smtp/outbound/mod.rs @@ -23,10 +23,11 @@ use std::sync::Arc; +use common::config::server::ServerProtocol; use tokio::sync::watch; use ::smtp::core::{SmtpAdminSessionManager, SmtpSessionManager, SMTP}; -use utils::config::{Config, ServerProtocol}; +use utils::config::Config; use super::add_test_certs; @@ -92,7 +93,7 @@ pub fn start_test_server(core: Arc, protocols: &[ServerProtocol]) -> watch server.spawn(smtp_manager.clone(), shutdown_rx) } ServerProtocol::Http => server.spawn(smtp_admin_manager.clone(), shutdown_rx), - ServerProtocol::Imap | ServerProtocol::Jmap | ServerProtocol::ManageSieve => { + ServerProtocol::Imap | ServerProtocol::ManageSieve => { unreachable!() } }; diff --git a/tests/src/smtp/outbound/mta_sts.rs b/tests/src/smtp/outbound/mta_sts.rs index c96f029f..259f6821 100644 --- a/tests/src/smtp/outbound/mta_sts.rs +++ b/tests/src/smtp/outbound/mta_sts.rs @@ -26,13 +26,19 @@ use std::{ time::{Duration, Instant}, }; +use common::{ + config::{ + server::ServerProtocol, + smtp::{queue::RequireOptional, report::AggregateFrequency, resolver::Policy}, + }, + expr::if_block::IfBlock, +}; use mail_auth::{ common::parse::TxtRecordParser, mta_sts::{MtaSts, ReportUri, TlsRpt}, report::tlsrpt::ResultType, MX, }; -use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, @@ -41,9 +47,8 @@ use crate::smtp::{ TestConfig, TestSMTP, }; use smtp::{ - config::{AggregateFrequency, RequireOptional}, core::{Session, SMTP}, - outbound::mta_sts::{lookup::STS_TEST_POLICY, Policy}, + outbound::mta_sts::lookup::STS_TEST_POLICY, reporting::PolicyType, }; @@ -59,13 +64,13 @@ async fn mta_sts_verify() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_mta_sts_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -73,12 +78,12 @@ async fn mta_sts_verify() { }], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "_smtp._tls.foobar.org", TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), Instant::now() + Duration::from_secs(10), @@ -87,9 +92,9 @@ async fn mta_sts_verify() { // Fail on missing MTA-STS record let mut local_qr = core.init_test_queue("smtp_mta_sts_local"); let mut rr = core.init_test_report(); - core.session.config.rcpt.relay = IfBlock::new(true); - core.queue.config.tls.mta_sts = IfBlock::new(RequireOptional::Require); - core.report.config.tls.send = IfBlock::new(AggregateFrequency::Weekly); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.queue.tls.mta_sts = IfBlock::new(RequireOptional::Require); + core.core.smtp.report.tls.send = IfBlock::new(AggregateFrequency::Weekly); let core = Arc::new(core); //let mut queue = Queue::default(); @@ -128,7 +133,7 @@ async fn mta_sts_verify() { ); // MTA-STS policy fetch failure - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "_mta-sts.foobar.org", MtaSts::parse(b"v=STSv1; id=policy_will_fail;").unwrap(), Instant::now() + Duration::from_secs(10), @@ -202,7 +207,7 @@ async fn mta_sts_verify() { remote_qr.assert_no_events(); // MTA-STS successful validation - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "_mta-sts.foobar.org", MtaSts::parse(b"v=STSv1; id=policy_will_work;").unwrap(), Instant::now() + Duration::from_secs(10), diff --git a/tests/src/smtp/outbound/smtp.rs b/tests/src/smtp/outbound/smtp.rs index 2c55839e..66d1f971 100644 --- a/tests/src/smtp/outbound/smtp.rs +++ b/tests/src/smtp/outbound/smtp.rs @@ -26,9 +26,9 @@ use std::{ time::{Duration, Instant}, }; +use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; use mail_auth::MX; use store::write::now; -use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, @@ -74,9 +74,9 @@ async fn smtp_delivery() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.extensions.dsn = IfBlock::new(true); - core.session.config.extensions.chunking = IfBlock::new(false); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.extensions.dsn = IfBlock::new(true); + core.core.smtp.session.extensions.chunking = IfBlock::new(false); let mut remote_qr = core.init_test_queue("smtp_delivery_remote"); let remote_core = Arc::new(core); let _rx = start_test_server(remote_core.clone(), &[ServerProtocol::Smtp]); @@ -84,7 +84,7 @@ async fn smtp_delivery() { // Add mock DNS entries let mut core = SMTP::test(); for domain in ["foobar.org", "foobar.net", "foobar.com"] { - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( domain, vec![MX { exchanges: vec![format!("mx1.{domain}"), format!("mx2.{domain}")], @@ -92,12 +92,12 @@ async fn smtp_delivery() { }], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( format!("mx1.{domain}"), vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(30), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( format!("mx2.{domain}"), vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(30), @@ -106,10 +106,10 @@ async fn smtp_delivery() { // Multiple delivery attempts let mut local_qr = core.init_test_queue("smtp_delivery_local"); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.rcpt.max_recipients = IfBlock::new(100); - core.session.config.extensions.dsn = IfBlock::new(true); - let config = &mut core.queue.config; + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.max_recipients = IfBlock::new(100); + core.core.smtp.session.extensions.dsn = IfBlock::new(true); + let config = &mut core.core.smtp.queue; config.retry = IfBlock::new(Duration::from_secs(1)); config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, {if = "rcpt_domain = 'foobar.com'", then = "[5s, 6s]"}, diff --git a/tests/src/smtp/outbound/throttle.rs b/tests/src/smtp/outbound/throttle.rs index dc0e8dff..7fea4966 100644 --- a/tests/src/smtp/outbound/throttle.rs +++ b/tests/src/smtp/outbound/throttle.rs @@ -27,9 +27,9 @@ use std::{ time::{Duration, Instant}, }; +use common::expr::if_block::IfBlock; use mail_auth::MX; use store::write::now; -use utils::config::if_block::IfBlock; use crate::smtp::{ inbound::TestQueueEvent, queue::manager::new_message, session::TestSession, ParseTestConfig, @@ -86,11 +86,11 @@ async fn throttle_outbound() { test_message.return_path_domain = "foobar.org".to_string(); let mut core = SMTP::test(); let mut local_qr = core.init_test_queue("smtp_throttle_outbound"); - core.session.config.rcpt.relay = IfBlock::new(true); - core.queue.config.throttle = THROTTLE.parse_queue_throttle(); - core.queue.config.retry = IfBlock::new(Duration::from_secs(86400)); - core.queue.config.notify = IfBlock::new(Duration::from_secs(86400)); - core.queue.config.expire = IfBlock::new(Duration::from_secs(86400)); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.queue.throttle = THROTTLE.parse_queue_throttle(); + core.core.smtp.queue.retry = IfBlock::new(Duration::from_secs(86400)); + core.core.smtp.queue.notify = IfBlock::new(Duration::from_secs(86400)); + core.core.smtp.queue.expire = IfBlock::new(Duration::from_secs(86400)); let core = Arc::new(core); let mut session = Session::test(core.clone()); @@ -105,7 +105,7 @@ async fn throttle_outbound() { // Throttle sender let span = tracing::info_span!("test"); let mut in_flight = vec![]; - let throttle = &core.queue.config.throttle; + let throttle = &core.core.smtp.queue.throttle; for t in &throttle.sender { core.is_allowed( t, @@ -215,7 +215,7 @@ async fn throttle_outbound() { assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for mx 'mx.test.org' - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "test.org", vec![MX { exchanges: vec!["mx.test.org".to_string()], @@ -223,7 +223,7 @@ async fn throttle_outbound() { }], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.test.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -251,7 +251,7 @@ async fn throttle_outbound() { in_flight.clear(); // Expect rate limit throttle for mx 'mx.test.net' - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "test.net", vec![MX { exchanges: vec!["mx.test.net".to_string()], @@ -259,7 +259,7 @@ async fn throttle_outbound() { }], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.test.net", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), diff --git a/tests/src/smtp/outbound/tls.rs b/tests/src/smtp/outbound/tls.rs index fcab4511..ee85437a 100644 --- a/tests/src/smtp/outbound/tls.rs +++ b/tests/src/smtp/outbound/tls.rs @@ -26,9 +26,12 @@ use std::{ time::{Duration, Instant}, }; +use common::{ + config::{server::ServerProtocol, smtp::queue::RequireOptional}, + expr::if_block::IfBlock, +}; use mail_auth::MX; use store::write::now; -use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{ inbound::TestMessage, @@ -36,10 +39,7 @@ use crate::smtp::{ session::{TestSession, VerifyResponse}, TestConfig, TestSMTP, }; -use smtp::{ - config::RequireOptional, - core::{Session, SMTP}, -}; +use smtp::core::{Session, SMTP}; #[tokio::test] #[serial_test::serial] @@ -53,14 +53,14 @@ async fn starttls_optional() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_starttls_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); - core.queue.config.hostname = IfBlock::new("badtls.foobar.org".to_string()); - core.resolvers.dns.mx_add( + core.core.smtp.queue.hostname = IfBlock::new("badtls.foobar.org".to_string()); + core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -68,7 +68,7 @@ async fn starttls_optional() { }], Instant::now() + Duration::from_secs(10), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -76,8 +76,8 @@ async fn starttls_optional() { // Retry on failed STARTTLS let mut local_qr = core.init_test_queue("smtp_starttls_local"); - core.session.config.rcpt.relay = IfBlock::new(true); - core.queue.config.tls.start = IfBlock::new(RequireOptional::Optional); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.queue.tls.start = IfBlock::new(RequireOptional::Optional); let core = Arc::new(core); let mut session = Session::test(core.clone()); diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index 9851a262..42385b85 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -26,8 +26,8 @@ use std::{ time::{Duration, Instant}, }; +use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; use mail_auth::MX; -use utils::config::{if_block::IfBlock, ServerProtocol}; use crate::smtp::{outbound::start_test_server, session::TestSession, TestConfig, TestSMTP}; use smtp::{ @@ -48,13 +48,13 @@ async fn concurrent_queue() { // Start test server let mut core = SMTP::test(); - core.session.config.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); let remote_qr = core.init_test_queue("smtp_concurrent_queue_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); // Add mock DNS entries let mut core = SMTP::test(); - core.resolvers.dns.mx_add( + core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -62,14 +62,14 @@ async fn concurrent_queue() { }], Instant::now() + Duration::from_secs(100), ); - core.resolvers.dns.ipv4_add( + core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); let local_qr = core.init_test_queue("smtp_concurrent_queue_local"); - core.session.config.rcpt.relay = IfBlock::new(true); - core.session.config.data.max_messages = IfBlock::new(200); + core.core.smtp.session.rcpt.relay = IfBlock::new(true); + core.core.smtp.session.data.max_messages = IfBlock::new(200); let core = Arc::new(core); let mut session = Session::test(core.clone()); @@ -105,8 +105,9 @@ async fn concurrent_queue() { assert_eq!(remote_messages.len(), 100); // Make sure local store is queue - core.shared - .default_data_store - .assert_is_empty(core.shared.default_blob_store.clone()) + core.core + .storage + .data + .assert_is_empty(core.core.storage.blob.clone()) .await; } diff --git a/tests/src/smtp/queue/dsn.rs b/tests/src/smtp/queue/dsn.rs index 86c8a9f5..84e73474 100644 --- a/tests/src/smtp/queue/dsn.rs +++ b/tests/src/smtp/queue/dsn.rs @@ -31,7 +31,6 @@ use crate::smtp::{ inbound::sign::TextConfigContext, ParseTestConfig, QueueReceiver, TestConfig, TestSMTP, }; use smtp::{ - config::ConfigContext, core::SMTP, queue::{Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status}, }; @@ -95,8 +94,8 @@ async fn generate_dsn() { // Load config let mut core = SMTP::test(); - core.shared.signers = ConfigContext::new().parse_signatures().signers; - let config = &mut core.queue.config.dsn; + core.core.storage.signers = ConfigContext::new().parse_signatures().signers; + let config = &mut core.core.smtp.queue.dsn; config.sign = "\"['rsa']\"".parse_if(); // Create temp dir for queue diff --git a/tests/src/smtp/queue/retry.rs b/tests/src/smtp/queue/retry.rs index a49c38f3..1d8c8e35 100644 --- a/tests/src/smtp/queue/retry.rs +++ b/tests/src/smtp/queue/retry.rs @@ -28,12 +28,12 @@ use crate::smtp::{ session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, TestSMTP, }; +use common::expr::if_block::IfBlock; use smtp::{ core::{Session, SMTP}, queue::{DeliveryAttempt, Event}, }; use store::write::now; -use utils::config::if_block::IfBlock; #[tokio::test] async fn queue_retry() { @@ -49,12 +49,12 @@ async fn queue_retry() { // Create temp dir for queue let mut qr = core.init_test_queue("smtp_queue_retry_test"); - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.relay = IfBlock::new(true); - let config = &mut core.session.config.extensions; + let config = &mut core.core.smtp.session.extensions; config.deliver_by = IfBlock::new(Duration::from_secs(86400)); config.future_release = IfBlock::new(Duration::from_secs(86400)); - let config = &mut core.queue.config; + let config = &mut core.core.smtp.queue; config.retry = r#""[1s, 2s, 3s]""#.parse_if(); config.notify = r#"[{if = "sender_domain = 'test.org'", then = "[1s, 2s]"}, {else = ['15h', '22h']}]"# diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs index fd014c5a..48692623 100644 --- a/tests/src/smtp/reporting/analyze.rs +++ b/tests/src/smtp/reporting/analyze.rs @@ -24,15 +24,12 @@ use std::{sync::Arc, time::Duration}; use crate::smtp::{inbound::TestQueueEvent, session::TestSession, TestConfig, TestSMTP}; -use smtp::{ - config::AddressMatch, - core::{Session, SMTP}, -}; +use common::{config::smtp::report::AddressMatch, expr::if_block::IfBlock}; +use smtp::core::{Session, SMTP}; use store::{ write::{ReportClass, ValueClass}, IterateParams, ValueKey, }; -use utils::config::if_block::IfBlock; #[tokio::test(flavor = "multi_thread")] async fn report_analyze() { @@ -40,11 +37,11 @@ async fn report_analyze() { // Create temp dir for queue let mut qr = core.init_test_queue("smtp_analyze_report_test"); - let config = &mut core.session.config.rcpt; + let config = &mut core.core.smtp.session.rcpt; config.relay = IfBlock::new(true); - let config = &mut core.session.config.data; + let config = &mut core.core.smtp.session.data; config.max_messages = IfBlock::new(1024); - let config = &mut core.report.config.analysis; + let config = &mut core.core.smtp.report.analysis; config.addresses = vec![ AddressMatch::StartsWith("reports@".to_string()), AddressMatch::EndsWith("@dmarc.foobar.org".to_string()), diff --git a/tests/src/smtp/reporting/dmarc.rs b/tests/src/smtp/reporting/dmarc.rs index c7ed7398..ccc9a638 100644 --- a/tests/src/smtp/reporting/dmarc.rs +++ b/tests/src/smtp/reporting/dmarc.rs @@ -27,24 +27,20 @@ use std::{ time::{Duration, Instant}, }; +use common::{config::smtp::report::AggregateFrequency, expr::if_block::IfBlock}; use mail_auth::{ common::parse::TxtRecordParser, dmarc::Dmarc, report::{ActionDisposition, Disposition, DmarcResult, Record, Report}, }; use store::write::QueueClass; -use utils::config::if_block::IfBlock; use crate::smtp::{ inbound::{sign::TextConfigContext, TestMessage}, session::VerifyResponse, ParseTestConfig, TestConfig, TestSMTP, }; -use smtp::{ - config::{AggregateFrequency, ConfigContext}, - core::SMTP, - reporting::DmarcEvent, -}; +use smtp::{core::SMTP, reporting::DmarcEvent}; #[tokio::test] async fn report_dmarc() { @@ -57,8 +53,8 @@ async fn report_dmarc() { // Create scheduler let mut core = SMTP::test(); - core.shared.signers = ConfigContext::new().parse_signatures().signers; - let config = &mut core.report.config; + core.core.storage.signers = ConfigContext::new().parse_signatures().signers; + let config = &mut core.core.smtp.report; config.dmarc_aggregate.sign = "\"['rsa']\"".parse_if(); config.dmarc_aggregate.max_size = IfBlock::new(4096); config.submitter = IfBlock::new("mx.example.org".to_string()); @@ -67,7 +63,7 @@ async fn report_dmarc() { config.dmarc_aggregate.contact_info = IfBlock::new("https://foobar.org/contact".to_string()); // Authorize external report for foobar.org - core.resolvers.dns.txt_add( + core.core.smtp.resolvers.dns.txt_add( "foobar.org._report._dmarc.foobar.net", Dmarc::parse(b"v=DMARC1;").unwrap(), Instant::now() + Duration::from_secs(10), diff --git a/tests/src/smtp/reporting/scheduler.rs b/tests/src/smtp/reporting/scheduler.rs index caccbdea..14c701a6 100644 --- a/tests/src/smtp/reporting/scheduler.rs +++ b/tests/src/smtp/reporting/scheduler.rs @@ -23,6 +23,7 @@ use std::sync::Arc; +use common::{config::smtp::report::AggregateFrequency, expr::if_block::IfBlock}; use mail_auth::{ common::parse::TxtRecordParser, dmarc::{Dmarc, URI}, @@ -30,11 +31,9 @@ use mail_auth::{ report::{ActionDisposition, Alignment, Disposition, DmarcResult, PolicyPublished, Record}, }; use store::write::QueueClass; -use utils::config::if_block::IfBlock; use crate::smtp::{TestConfig, TestSMTP}; use smtp::{ - config::AggregateFrequency, core::SMTP, reporting::{dmarc::DmarcFormat, DmarcEvent, PolicyType, TlsEvent}, }; @@ -51,7 +50,7 @@ async fn report_scheduler() { // Create scheduler let mut core = SMTP::test(); let qr = core.init_test_queue("smtp_report_queue_test"); - let config = &mut core.report.config; + let config = &mut core.core.smtp.report; config.dmarc_aggregate.max_size = IfBlock::new(500); config.tls.max_size = IfBlock::new(550); diff --git a/tests/src/smtp/reporting/tls.rs b/tests/src/smtp/reporting/tls.rs index a09cfd62..bccaa7a9 100644 --- a/tests/src/smtp/reporting/tls.rs +++ b/tests/src/smtp/reporting/tls.rs @@ -23,6 +23,7 @@ use std::{io::Read, sync::Arc, time::Duration}; +use common::{config::smtp::report::AggregateFrequency, expr::if_block::IfBlock}; use mail_auth::{ common::parse::TxtRecordParser, flate2::read::GzDecoder, @@ -30,7 +31,6 @@ use mail_auth::{ report::tlsrpt::{FailureDetails, PolicyType, ResultType, TlsReport}, }; use store::write::QueueClass; -use utils::config::if_block::IfBlock; use crate::smtp::{ inbound::{sign::TextConfigContext, TestMessage}, @@ -38,7 +38,6 @@ use crate::smtp::{ ParseTestConfig, TestConfig, TestSMTP, }; use smtp::{ - config::{AggregateFrequency, ConfigContext}, core::SMTP, reporting::{tls::TLS_HTTP_REPORT, TlsEvent}, }; @@ -55,8 +54,8 @@ async fn report_tls() { // Create scheduler let mut core = SMTP::test(); - core.shared.signers = ConfigContext::new().parse_signatures().signers; - let config = &mut core.report.config; + core.core.storage.signers = ConfigContext::new().parse_signatures().signers; + let config = &mut core.core.smtp.report; config.tls.sign = "\"['rsa']\"".parse_if(); config.tls.max_size = IfBlock::new(1532); config.submitter = IfBlock::new("mx.example.org".to_string()); diff --git a/tests/src/smtp/session.rs b/tests/src/smtp/session.rs index d656492d..281d7730 100644 --- a/tests/src/smtp/session.rs +++ b/tests/src/smtp/session.rs @@ -23,6 +23,10 @@ use std::{borrow::Cow, path::PathBuf, sync::Arc}; +use common::{ + config::server::ServerProtocol, + listener::{limiter::ConcurrencyLimiter, ServerInstance, SessionStream, TcpAcceptor}, +}; use rustls::{server::ResolvesServerCert, ServerConfig}; use tokio::{ io::{AsyncRead, AsyncWrite}, @@ -31,10 +35,6 @@ use tokio::{ use smtp::core::{Session, SessionAddress, SessionData, SessionParameters, State, SMTP}; use tokio_rustls::TlsAcceptor; -use utils::{ - config::ServerProtocol, - listener::{limiter::ConcurrencyLimiter, ServerInstance, SessionStream, TcpAcceptor}, -}; use super::TestConfig; @@ -132,6 +132,7 @@ impl TestSession for Session { ), params: SessionParameters::default(), in_flight: vec![], + hostname: "localhost".to_string(), } } @@ -360,10 +361,7 @@ impl TestServerInstance for ServerInstance { fn test_with_shutdown(shutdown_rx: watch::Receiver) -> Self { Self { id: "smtp".to_string(), - listener_id: 1, - hostname: "mx.example.org".to_string(), protocol: ServerProtocol::Smtp, - data: "220 mx.example.org at your service.\r\n".to_string(), acceptor: TcpAcceptor::Tls(TlsAcceptor::from(Arc::new( ServerConfig::builder() .with_no_client_auth() diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index b59ea8fe..cd2b15cb 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -23,7 +23,6 @@ use ahash::AHashMap; use store::{ - config::ConfigStore, write::{blob::BlobQuota, now, BatchBuilder, BlobOp}, BlobClass, BlobStore, Serialize, }; diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index 7cd986cb..fef66f4f 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -23,7 +23,7 @@ use std::time::Duration; -use store::{config::ConfigStore, LookupStore}; +use store::LookupStore; use utils::config::{Config, Rate}; use crate::store::{TempDir, CONFIG}; diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 419dd175..1f0ade4c 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -29,7 +29,7 @@ pub mod query; use std::io::Read; -use store::{config::ConfigStore, FtsStore}; +use store::FtsStore; use utils::config::Config; pub struct TempDir {