diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 782c5c2..74d7d61 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,14 +59,24 @@ jobs: run: echo "LIBCLANG_PATH=$((gcm clang).source -replace "clang.exe")" >> $env:GITHUB_ENV if: ${{ contains(matrix.host_os, 'windows') }} - - name: Install LLVM and Clang (Linux) + - name: Install protoc, LLVM and Clang (Linux) if: ${{ contains(matrix.host_os, 'ubuntu') }} run: | sudo apt-get update -y + sudo apt-get install protobuf-compiler sudo wget https://apt.llvm.org/llvm.sh sudo chmod +x llvm.sh sudo ./llvm.sh 14 all + - name: Install protoc (MacOs) + if: ${{ contains(matrix.host_os, 'macos') }} + run: | + brew install protobuf + + - name: Install protoc (Windows) + if: ${{ contains(matrix.host_os, 'windows') }} + uses: arduino/setup-protoc@v1 + - name: Install ARM64 Linux build tools if: ${{ matrix.target == 'aarch64-unknown-linux-gnu' }} run: | @@ -99,7 +109,7 @@ jobs: - name: XCode Version if: ${{ matrix.target == 'aarch64-apple-darwin' }} run: | - sudo xcode-select -s /Applications/Xcode_12.4.app && + sudo xcode-select -s /Applications/Xcode_13.4.app && sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/* - name: ARM64 Windows setup diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 823b017..0aee426 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,6 +38,11 @@ jobs: - name: Checkout uses: actions/checkout@v1 + - name: Install protobuf-compiler + run: | + sudo apt-get update -y + sudo apt-get install protobuf-compiler + - name: Install Rust uses: actions-rs/toolchain@v1 with: @@ -51,33 +56,3 @@ jobs: command: test args: --all - - name: Database Tests - uses: actions-rs/cargo@v1 - with: - command: test - args: store_tests --all - - - name: JMAP Core Tests - uses: actions-rs/cargo@v1 - with: - command: test - args: jmap_core_tests -- --ignored - - - name: JMAP Mail Tests - uses: actions-rs/cargo@v1 - with: - command: test - args: jmap_mail_tests -- --ignored - - - name: Stress Tests - uses: actions-rs/cargo@v1 - with: - command: test - args: jmap_stress_tests -- --ignored - - - name: Cluster Tests - uses: actions-rs/cargo@v1 - with: - command: test - args: cluster_tests -- --ignored - diff --git a/Cargo.toml b/Cargo.toml index f4a006e..33f917c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,6 +49,8 @@ reqwest = { version = "0.11", default-features = false, features = ["rustls-tls" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" num_cpus = "1.15.0" + +[target.'cfg(unix)'.dependencies] privdrop = "0.5.3" [dev-dependencies] diff --git a/Dockerfile b/Dockerfile index eaf5840..44d71fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ RUN apt-get update && \ build-essential \ cmake \ clang \ - curl + curl \ + protobuf-compiler ENV RUSTUP_HOME=/opt/rust/rustup \ PATH=/home/root/.cargo/bin:/opt/rust/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN curl https://sh.rustup.rs -sSf | \ @@ -21,7 +22,6 @@ FROM chef AS planner COPY Cargo.toml . COPY Cargo.lock . COPY src/ src/ -COPY components/ components/ COPY resources/ resources/ RUN cargo chef prepare --recipe-path recipe.json @@ -31,7 +31,6 @@ RUN cargo chef cook --release --recipe-path recipe.json COPY Cargo.toml . COPY Cargo.lock . COPY src/ src/ -COPY components/ components/ COPY resources/ resources/ RUN cargo build --release diff --git a/README.md b/README.md index b83375d..ee9f111 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![](https://img.shields.io/twitter/follow/stalwartlabs?style=flat)](https://twitter.com/stalwartlabs) **Stalwart SMTP** is a modern SMTP server developed in Rust with a focus on security, speed, and extensive configurability. -It features built-in DMARC, DKIM, SPF and ARC support for message and sender authentication, strong transport security through DANE, MTA-STS, SMTP TLS reporting and offers great flexibility and customization thanks to its configurable rules and native support for Sieve scripts. +It features built-in DMARC, DKIM, SPF and ARC support for message authentication, strong transport security through DANE, MTA-STS and SMTP TLS reporting, and offers great flexibility and customization thanks to its dynamic configuration rules and native support for Sieve scripts. Key features: @@ -24,17 +24,14 @@ Key features: - Inbound Filtering and Throttling: - Sieve scripting language with support for all [registered extensions](https://www.iana.org/assignments/sieve-extensions/sieve-extensions.xhtml). - Filtering, modification and removal of MIME parts or headers. - - DNS block lists (**DNSBL**). - - Greylisting. - - Inbound concurrency and rate limiting. + - DNS block lists (**DNSBL**) & Greylisting. + - Inbound concurrency & rate limiting. - Integration with external content filtering systems such as SpamAssassin and ClamAV. - Flexible Queues: - - Unlimited virtual queues. + - Unlimited virtual queues with custom routing rules. - Delayed delivery with `FUTURERELEASE` and `DELIVERBY` extensions support. - Priority delivery with `MT-PRIORITY` extension support. - - Quotas. - - Outbound throttling. - - Custom routing rules. + - Outbound throttling & Disk quotas. - Logging and Reporting: - Detailed logging of SMTP transactions and events, including delivery attempts, errors, and policy violations. - Integration with **OpenTelemetry** to enable monitoring, tracing, and performance analysis of SMTP server operations. @@ -60,7 +57,7 @@ You may also [compile Stalwart SMTP from the source](https://stalw.art/smtp/deve If you are having problems running Stalwart SMTP, you found a bug or just have a question, do not hesitate to reach us on [Github Discussions](https://github.com/stalwartlabs/smtp-server/discussions), -[Reddit](https://www.reddit.com/r/stalwartlabs) or [Discord](https://discord.gg/jtgtCNj66U). +[Reddit](https://www.reddit.com/r/stalwartlabs) or [Discord](https://discord.gg/9dXkHzCk). Additionally you may become a sponsor to obtain priority support from Stalwart Labs Ltd. ## Documentation @@ -72,45 +69,43 @@ Table of Contents - [Windows](https://stalw.art/smtp/get-started/windows/) - [Docker](https://stalw.art/smtp/get-started/docker/) - Configuration - - Overview - - General - - Listeners - - Rules - - Scripting - - Certificates - - Remote - - Databases - - Lists - - Resolver - - Logging -- Inbound - - Overview - - Connect - - Extensions - - Ehlo - - Authentication - - Mail-From - - Data -- Outbound - - Overview - - Schedule - - Queue strategy - - Throttling - - Quotas -- Authentication - - Overview - - Signatures - - DKIM - - SPF - - ARC - - DMARC - - IpRev - - DNSBL + - [Overview](https://stalw.art/smtp/settings/overview) + - [Configuration Rules](https://stalw.art/smtp/settings/rules) + - [General settings](https://stalw.art/smtp/settings/general) + - [Remote hosts](https://stalw.art/smtp/settings/remote) + - [Databases](https://stalw.art/smtp/settings/database) + - [Local Lists](https://stalw.art/smtp/settings/list) + - [Tracing & Logging](https://stalw.art/smtp/settings/tracing) +- Inbound settings + - [Listeners](https://stalw.art/smtp/inbound/listeners) + - [Sessions](https://stalw.art/smtp/inbound/session) + - [EHLO Stage](https://stalw.art/smtp/inbound/ehlo) + - [MAIL Stage](https://stalw.art/smtp/inbound/mail) + - [RCPT Stage](https://stalw.art/smtp/inbound/rcpt) + - [DATA Stage](https://stalw.art/smtp/inbound/data) + - [AUTH Stage](https://stalw.art/smtp/inbound/auth) + - [DNSBLs](https://stalw.art/smtp/inbound/dnsbl) + - [Sieve Scripting](https://stalw.art/smtp/inbound/sieve) + - [Throttling](https://stalw.art/smtp/inbound/throttle) +- Outbound settings + - [Queues](https://stalw.art/smtp/outbound/queue) + - [Transport & Routing](https://stalw.art/smtp/outbound/transport) + - [TLS Security](https://stalw.art/smtp/outbound/tls) + - [Throttling](https://stalw.art/smtp/outbound/throttle) + - [Quotas](https://stalw.art/smtp/outbound/quota) + - [DNS](https://stalw.art/smtp/outbound/dns) +- Email Authentication + - [DKIM](https://stalw.art/smtp/auth/dkim) + - [SPF](https://stalw.art/smtp/auth/spf) + - [ARC](https://stalw.art/smtp/auth/arc) + - [DMARC](https://stalw.art/smtp/auth/dmarc) + - [Reverse IP](https://stalw.art/smtp/auth/iprev) + - [Report Analysis](https://stalw.art/smtp/auth/analysis) - Management - - Overview - - Configuration - - Queue - - Reports + - [API](https://stalw.art/smtp/management/api) + - [CLI](https://stalw.art/smtp/management/cli) + - [Queue](https://stalw.art/smtp/management/queue) + - [Reports](https://stalw.art/smtp/management/reports) - Development - [Compiling](https://stalw.art/smtp/development/compile/) - [Tests](https://stalw.art/smtp/development/test/) @@ -127,7 +122,8 @@ The following major features and enhancements are planned for Stalwart SMTP: ## Testing & Fuzzing -To run the test suite execute: +The base tests perform protocol compliance tests as well as basic functionality testing on different functions across the Stalwart SMTP code base. +To run the base test suite execute: ```bash cargo test @@ -137,6 +133,12 @@ To run the fuzz tests please refer to the Stalwart libraries that handle parsing [mail-parser](https://github.com/stalwartlabs/mail-parser), [mail-auth](https://github.com/stalwartlabs/mail-auth) and [sieve-rs](https://github.com/stalwartlabs/sieve). +## Funding + +Part of the development of this project was funded through the [NGI0 Entrust Fund](https://nlnet.nl/entrust), a fund established by [NLnet](https://nlnet.nl/) with financial support from the European Commission's [Next Generation Internet](https://ngi.eu/) programme, under the aegis of DG Communications Networks, Content and Technology under grant agreement No 101069594. + +If you find the project useful you can help by [becoming a sponsor](https://github.com/sponsors/stalwartlabs). Thank you! + ## License Licensed under the terms of the [GNU Affero General Public License](https://www.gnu.org/licenses/agpl-3.0.en.html) as published by diff --git a/resources/config/config.toml b/resources/config/config.toml index 9007c2d..82ce3f2 100644 --- a/resources/config/config.toml +++ b/resources/config/config.toml @@ -230,7 +230,7 @@ data = "10m" mta-sts = "2m" [[queue.quota]] -#match = {if = "remote-ip", eq = "10.0.0.1"} +#match = {if = "sender-domain", eq = "foobar.org"} #key = ["rcpt"] messages = 100000 size = 10737418240 # 10gb @@ -402,7 +402,55 @@ nested-includes = 5 duplicate-expiry = "7d" [sieve.scripts] -#ehlo = "" +# Note: These scripts are included here for demonstration purposes. +# They should not be used in their current form. +connect = ''' + require ["variables", "extlists", "reject"]; + + if string :list "${env.remote_ip}" "list/blocked-ips" { + reject "Your IP '${env.remote_ip}' is not welcomed here."; + } +''' +ehlo = ''' + require ["variables", "extlists", "reject"]; + + if string :list "${env.helo_domain}" "list/blocked-domains" { + reject "551 5.1.1 Your domain '${env.helo_domain}' has been blacklisted."; + } +''' +mail = ''' + require ["variables", "envelope", "reject"]; + + if envelope :localpart :is "from" "known_spammer" { + reject "We do not accept SPAM."; + } +''' +rcpt = ''' + require ["variables", "vnd.stalwart.execute", "envelope", "reject"]; + + set "triplet" "${env.remote_ip}.${envelope.from}.${envelope.to}"; + + if not execute :query "SELECT EXISTS(SELECT 1 FROM greylist WHERE addr=? LIMIT 1)" ["${triplet}"] { + execute :query "INSERT INTO greylist (addr) VALUES (?)" ["${triplet}"]; + reject "422 4.2.2 Greylisted, please try again in a few moments."; + } +''' +data = ''' + require ["envelope", "variables", "replace", "mime", "foreverypart", "editheader", "extracttext"]; + + if envelope :domain :is "to" "foobar.net" { + set "counter" "a"; + foreverypart { + if header :mime :contenttype "content-type" "text/html" { + extracttext :upper "text_content"; + replace "${text_content}"; + } + set :length "part_num" "${counter}"; + addheader :last "X-Part-Number" "${part_num}"; + set "counter" "${counter}a"; + } + } +''' [management.lookup] auth = ["list/admin"] @@ -410,6 +458,8 @@ auth = ["list/admin"] [list] domains = ["__DOMAIN__"] admin = ["admin:__ADMIN_PASS__"] +#blocked-ips = ["10.0.0.1"] +#blocked-domains = ["mail.spammer.com"] #users = "file:///usr/local/stalwart-smtp/etc/users.txt" [certificate."default"] diff --git a/src/config/condition.rs b/src/config/condition.rs index ef8fc8d..794ef1f 100644 --- a/src/config/condition.rs +++ b/src/config/condition.rs @@ -29,8 +29,7 @@ use crate::config::StringMatch; use super::{ utils::{AsKey, ParseKey, ParseValue}, - Condition, ConditionMatch, Conditions, Config, ConfigContext, EnvelopeKey, - IpAddrMask, + Condition, ConditionMatch, Conditions, Config, ConfigContext, EnvelopeKey, IpAddrMask, }; impl Config { @@ -58,14 +57,13 @@ impl Config { } else { return Err(format!( "Multiple operations found for condition {prefix:?}.", - )); } } } if op_str.is_empty() { - return Err(format!("Missing operation for condition {prefix:?}." )); + return Err(format!("Missing operation for condition {prefix:?}.")); } else if ["any-of", "all-of", "none-of"].contains(&op_str) { stack.push(( std::mem::replace( @@ -113,7 +111,6 @@ impl Config { if !available_keys.contains(&key) { return Err(format!( "Envelope key {key:?} is not available in this context for property {prefix:?}", - )); } @@ -129,18 +126,16 @@ impl Config { "eq" | "equal-to" | "ne" | "not-equal-to" => { (MatchType::Equal, op_str == "ne" || op_str == "not-equal-to") } - "in-list" | "not-in-list" => { - (MatchType::Lookup, op_str == "not-in-list") + "in-list" | "not-in-list" => (MatchType::Lookup, op_str == "not-in-list"), + "matches" | "not-matches" => (MatchType::Regex, op_str.starts_with("not-")), + "starts-with" | "not-starts-with" => { + (MatchType::StartsWith, op_str == "not-starts-with") } - "matches" | "not-matches" => { - (MatchType::Regex, op_str.starts_with("not-")) + "ends-with" | "not-ends-with" => { + (MatchType::EndsWith, op_str == "not-ends-with") } - "starts-with" | "not-starts-with" => (MatchType::StartsWith, op_str == "not-starts-with"), - "ends-with" | "not-ends-with" => (MatchType::EndsWith, op_str == "not-ends-with"), _ => { - return Err(format!( - "Invalid operation {op_str:?} for key {prefix:?}." - )); + return Err(format!("Invalid operation {op_str:?} for key {prefix:?}.")); } }; @@ -180,33 +175,38 @@ impl Config { | EnvelopeKey::LocalIp | EnvelopeKey::RemoteIp, _, - ) => { - match op { - MatchType::Equal => ConditionMatch::String(StringMatch::Equal(value_str.to_string())), - MatchType::StartsWith => ConditionMatch::String(StringMatch::StartsWith(value_str.to_string())), - MatchType::EndsWith => ConditionMatch::String(StringMatch::EndsWith(value_str.to_string())), - MatchType::Regex => ConditionMatch::Regex(Regex::new(value_str).map_err(|err| { + ) => match op { + MatchType::Equal => { + ConditionMatch::String(StringMatch::Equal(value_str.to_string())) + } + MatchType::StartsWith => { + ConditionMatch::String(StringMatch::StartsWith(value_str.to_string())) + } + MatchType::EndsWith => { + ConditionMatch::String(StringMatch::EndsWith(value_str.to_string())) + } + MatchType::Regex => { + ConditionMatch::Regex(Regex::new(value_str).map_err(|err| { format!( "Failed to compile regular expression {:?} for key {:?}: {}.", value_str, (&prefix, value_str).as_key(), err ) - })?), - MatchType::Lookup => { - if let Some(list) = ctx.lookup.get(value_str) { - ConditionMatch::Lookup(list.clone()) - } else { - return Err(format!( - "Lookup {:?} not found for property {:?}.", - value_str, - (&prefix, value_str).as_key() - )); - } - }, + })?) } - - } + MatchType::Lookup => { + if let Some(list) = ctx.lookup.get(value_str) { + ConditionMatch::Lookup(list.clone()) + } else { + return Err(format!( + "Lookup {:?} not found for property {:?}.", + value_str, + (&prefix, value_str).as_key() + )); + } + } + }, _ => { return Err(format!( "Invalid 'op'/'value' combination for key {:?}.", @@ -347,10 +347,13 @@ mod tests { use ahash::AHashMap; - use crate::{config::{ - Condition, ConditionMatch, Conditions, Config, ConfigContext, EnvelopeKey, - IpAddrMask, Server, StringMatch, - }, lookup::Lookup}; + use crate::{ + config::{ + Condition, ConditionMatch, Conditions, Config, ConfigContext, EnvelopeKey, IpAddrMask, + Server, StringMatch, + }, + lookup::Lookup, + }; #[test] fn parse_conditions() { @@ -397,7 +400,9 @@ mod tests { conditions: vec![ Condition::Match { key: EnvelopeKey::SenderDomain, - value: ConditionMatch::String(StringMatch::StartsWith("example".to_string())), + value: ConditionMatch::String(StringMatch::StartsWith( + "example".to_string(), + )), not: false, }, Condition::JumpIfFalse { positions: 1 }, @@ -415,7 +420,9 @@ mod tests { conditions: vec![ Condition::Match { key: EnvelopeKey::RecipientDomain, - value: ConditionMatch::String(StringMatch::Equal("example.org".to_string())), + value: ConditionMatch::String(StringMatch::Equal( + "example.org".to_string(), + )), not: false, }, Condition::JumpIfTrue { positions: 9 }, @@ -430,13 +437,17 @@ mod tests { Condition::JumpIfTrue { positions: 7 }, Condition::Match { key: EnvelopeKey::Recipient, - value: ConditionMatch::String(StringMatch::StartsWith("no-reply@".to_string())), + value: ConditionMatch::String(StringMatch::StartsWith( + "no-reply@".to_string(), + )), not: false, }, Condition::JumpIfFalse { positions: 5 }, Condition::Match { key: EnvelopeKey::Sender, - value: ConditionMatch::String(StringMatch::EndsWith("@domain.org".to_string())), + value: ConditionMatch::String(StringMatch::EndsWith( + "@domain.org".to_string(), + )), not: false, }, Condition::JumpIfFalse { positions: 3 }, @@ -457,7 +468,7 @@ mod tests { ]); for (key, rule) in expected_rules { - assert_eq!(Some(rule), conditions.remove(&key), "failed for {key}" ); + assert_eq!(Some(rule), conditions.remove(&key), "failed for {key}"); } } } diff --git a/src/config/queue.rs b/src/config/queue.rs index 77226f0..d79625d 100644 --- a/src/config/queue.rs +++ b/src/config/queue.rs @@ -170,7 +170,7 @@ impl Config { .unwrap_or_else(|| IfBlock::new("Mail Delivery Subsystem".to_string())), address: self .parse_if_block("report.dsn.from-address", ctx, &sender_envelope_keys)? - .unwrap_or_else(|| IfBlock::new(format!("MAILER-DAEMON@{default_hostname}" ))), + .unwrap_or_else(|| IfBlock::new(format!("MAILER-DAEMON@{default_hostname}"))), sign: self .parse_if_block::>("report.dsn.sign", ctx, &sender_envelope_keys)? .unwrap_or_default() @@ -178,14 +178,14 @@ impl Config { }, management_lookup: if let Some(lookup) = self.value("management.auth.lookup") { ctx.lookup - .get(lookup) - .ok_or_else(|| format!( - "Lookup {lookup:?} not found for key \"management.auth.lookup\"." - ))? - .clone() + .get(lookup) + .ok_or_else(|| { + format!("Lookup {lookup:?} not found for key \"management.auth.lookup\".") + })? + .clone() } else { Arc::new(Lookup::default()) - } + }, }; if config.retry.has_empty_list() { @@ -378,7 +378,6 @@ impl IfBlock> { .ok_or_else(|| { format!( "Host {then:?} not found for property \"queue.next-hop\".", - ) })? .into(), diff --git a/src/main.rs b/src/main.rs index b5655fe..03ad194 100644 --- a/src/main.rs +++ b/src/main.rs @@ -60,6 +60,9 @@ async fn main() -> std::io::Result<()> { config .parse_remote_hosts(&mut config_context) .failed("Configuration error"); + config + .parse_databases(&mut config_context) + .failed("Configuration error"); config .parse_lists(&mut config_context) .failed("Configuration error"); @@ -262,7 +265,7 @@ async fn main() -> std::io::Result<()> { fn enable_tracing(config: &Config) -> stalwart_smtp::config::Result> { let level = config.value("global.tracing.level").unwrap_or("info"); let env_filter = EnvFilter::builder() - .parse(format!("smtp_server={}", level)) + .parse(format!("stalwart_smtp={}", level)) .failed("Failed to log level"); match config.value("global.tracing.method").unwrap_or_default() { "log" => {