From 875b1fa744aa172d0ec72bc66ec6ff1b8d484be7 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sat, 2 Mar 2024 12:21:03 +0100 Subject: [PATCH] Store incoming reports in the data store --- Cargo.lock | 609 ++++--- crates/cli/Cargo.toml | 1 + crates/cli/src/main.rs | 19 +- crates/cli/src/modules/mod.rs | 6 + crates/cli/src/modules/queue.rs | 224 ++- crates/cli/src/modules/report.rs | 186 ++- crates/jmap/src/api/admin.rs | 6 +- crates/smtp/Cargo.toml | 4 +- crates/smtp/src/config/mod.rs | 12 +- crates/smtp/src/config/report.rs | 6 +- crates/smtp/src/core/management.rs | 1434 +++++++++-------- crates/smtp/src/reporting/analysis.rs | 149 +- crates/smtp/src/reporting/dkim.rs | 2 +- crates/smtp/src/reporting/dmarc.rs | 161 +- crates/smtp/src/reporting/mod.rs | 1 + crates/smtp/src/reporting/scheduler.rs | 6 +- crates/smtp/src/reporting/tls.rs | 318 ++-- crates/smtp/src/scripts/functions/unicode.rs | 5 +- crates/store/Cargo.toml | 4 +- .../store/src/backend/foundationdb/write.rs | 2 +- crates/store/src/backend/mysql/write.rs | 2 +- crates/store/src/backend/postgres/write.rs | 2 +- crates/store/src/backend/rocksdb/write.rs | 2 +- crates/store/src/backend/sqlite/write.rs | 2 +- crates/store/src/config.rs | 2 +- crates/store/src/dispatch/lookup.rs | 2 +- crates/store/src/dispatch/store.rs | 43 +- crates/store/src/write/key.rs | 16 +- crates/store/src/write/mod.rs | 8 + crates/store/src/write/purge.rs | 8 +- crates/utils/Cargo.toml | 4 +- resources/config/smtp/report.toml | 2 +- tests/src/smtp/management/mod.rs | 23 +- tests/src/smtp/management/queue.rs | 101 +- tests/src/smtp/management/report.rs | 56 +- tests/src/smtp/mod.rs | 2 +- tests/src/smtp/reporting/analyze.rs | 71 +- tests/src/smtp/reporting/dmarc.rs | 2 +- tests/src/smtp/reporting/tls.rs | 22 +- tests/src/store/lookup.rs | 8 +- 40 files changed, 1992 insertions(+), 1541 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93fc6277..ac50721b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -29,9 +29,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher 0.4.4", @@ -81,9 +81,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" +checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b" dependencies = [ "cfg-if", "getrandom", @@ -125,9 +125,9 @@ dependencies = [ [[package]] name = "anstream" -version = "0.6.11" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", @@ -173,9 +173,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.79" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080e9890a082662b09c1ad45f567faeeb47f22b5fb23895fbe1e651e718e25ca" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "arc-swap" @@ -222,8 +222,24 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" dependencies = [ - "asn1-rs-derive", - "asn1-rs-impl", + "asn1-rs-derive 0.4.0", + "asn1-rs-impl 0.1.0", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "asn1-rs" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ad1373757efa0f70ec53939aabc7152e1591cb485208052993070ac8d2429d" +dependencies = [ + "asn1-rs-derive 0.5.0", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -241,7 +257,19 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.109", - "synstructure", + "synstructure 0.12.6", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7378575ff571966e99a744addeff0bff98b8ada0dedf1956d59e634db95eaac1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", + "synstructure 0.13.1", ] [[package]] @@ -255,6 +283,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "async-compression" version = "0.4.6" @@ -276,7 +315,7 @@ checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -298,7 +337,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -309,7 +348,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -491,7 +530,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.48", + "syn 2.0.52", "which", ] @@ -512,7 +551,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -654,7 +693,7 @@ dependencies = [ "proc-macro-crate 3.1.0", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "syn_derive", ] @@ -679,9 +718,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" [[package]] name = "bytecheck" @@ -774,11 +813,10 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.83" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +checksum = "02f341c093d19155a6e41631ce5971aac4e9a868262212153124c15fa22d1cdc" dependencies = [ - "jobserver", "libc", ] @@ -844,7 +882,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -880,9 +918,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80c21025abd42669a92efc996ef13cfb2c5c627858421ea58d5c3b331a6c134f" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", "clap_derive", @@ -890,9 +928,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.0" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458bf1f341769dfcf849846f65dffdf9146daa56bcd2a47cb4e1de9915567c99" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstream", "anstyle", @@ -909,7 +947,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1050,9 +1088,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176dc175b78f56c0f321911d9c8eb2b77a78a4860b9c19db83835fea1a46649b" +checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" dependencies = [ "crossbeam-utils", ] @@ -1185,7 +1223,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1200,12 +1238,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.5" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc5d6b04b3fd0ba9926f945895de7d806260a2d7431ba82e7edaecb043c4c6b8" +checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391" dependencies = [ - "darling_core 0.20.5", - "darling_macro 0.20.5", + "darling_core 0.20.8", + "darling_macro 0.20.8", ] [[package]] @@ -1224,16 +1262,16 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.20.5" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e48a959bcd5c761246f5d090ebc2fbf7b9cd527a492b07a67510c108f1e7e3" +checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim 0.10.0", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1249,13 +1287,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.20.5" +version = "0.20.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1545d67a2149e1d93b7e5c7752dce5a7426eb5d1357ddcfd89336b94444f77" +checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f" dependencies = [ - "darling_core 0.20.5", + "darling_core 0.20.8", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1321,9 +1359,12 @@ dependencies = [ [[package]] name = "decancer" -version = "1.6.5" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "080b09f6adad25c23d8c47c54e52e59b0dc09d079c4b23e0f147dac440359d0d" +checksum = "b49785bd69347eb1a5c8a3339adf11d952ecb16b19a361aa156e249ebf73c928" +dependencies = [ + "paste", +] [[package]] name = "der" @@ -1342,7 +1383,21 @@ version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" dependencies = [ - "asn1-rs", + "asn1-rs 0.5.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der-parser" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +dependencies = [ + "asn1-rs 0.6.1", "displaydoc", "nom", "num-bigint", @@ -1382,12 +1437,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "digest" version = "0.9.0" @@ -1413,7 +1462,7 @@ dependencies = [ name = "directory" version = "0.1.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "argon2", "async-trait", "deadpool", @@ -1493,7 +1542,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1526,9 +1575,9 @@ dependencies = [ [[package]] name = "dyn-clone" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d" +checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "eax" @@ -1695,7 +1744,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1896,7 +1945,7 @@ checksum = "83c8d52fe8b46ab822b4decdcc0d6d85aeedfc98f0d52ba2bd4aec4a97807516" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "try_map", ] @@ -1934,7 +1983,7 @@ checksum = "b0fa992f1656e1707946bbba340ad244f0814009ef8c0118eb7b658395f19a2e" dependencies = [ "frunk_proc_macro_helpers", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1946,7 +1995,7 @@ dependencies = [ "frunk_core", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -1958,7 +2007,7 @@ dependencies = [ "frunk_core", "frunk_proc_macro_helpers", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2023,7 +2072,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -2153,7 +2202,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.2.3", + "indexmap 2.2.5", "slab", "tokio", "tokio-util", @@ -2172,7 +2221,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.2.3", + "indexmap 2.2.5", "slab", "tokio", "tokio-util", @@ -2194,15 +2243,15 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "allocator-api2", ] [[package]] name = "hashlink" -version = "0.8.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +checksum = "692eaaf7f7607518dd3cef090f1474b61edc5301d8012f09579920df68b725ee" dependencies = [ "hashbrown 0.14.3", ] @@ -2215,9 +2264,9 @@ checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" -version = "0.3.5" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c62115964e08cb8039170eb33c1d0e2388a256930279edca206fff675f82c3" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "hex" @@ -2415,7 +2464,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -2424,9 +2473,9 @@ dependencies = [ [[package]] name = "hyper" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5aa53871fc917b1a9ed87b683a5d86db645e23acb32c2e0785a353e522fb75" +checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" dependencies = [ "bytes", "futures-channel", @@ -2438,6 +2487,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", + "smallvec", "tokio", ] @@ -2477,9 +2527,9 @@ dependencies = [ "futures-util", "http 1.0.0", "http-body 1.0.0", - "hyper 1.1.0", + "hyper 1.2.0", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.5.6", "tokio", ] @@ -2551,7 +2601,7 @@ checksum = "029d73f573d8e8d63e6d5020011d3255b28c3ba85d6cf870a07184ed23de9284" name = "imap" version = "0.6.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "dashmap", "directory", "imap_proto", @@ -2564,7 +2614,7 @@ dependencies = [ "parking_lot", "rand", "rustls 0.22.2", - "rustls-pemfile 2.0.0", + "rustls-pemfile 2.1.1", "store", "tokio", "tokio-rustls 0.25.0", @@ -2576,7 +2626,7 @@ dependencies = [ name = "imap_proto" version = "0.1.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "chrono", "jmap_proto", "mail-parser", @@ -2596,9 +2646,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.3" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2651,7 +2701,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ - "socket2 0.5.5", + "socket2 0.5.6", "widestring", "windows-sys 0.48.0", "winreg", @@ -2683,6 +2733,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -2754,7 +2813,7 @@ dependencies = [ "futures-util", "hkdf", "http-body-util", - "hyper 1.1.0", + "hyper 1.2.0", "hyper-util", "jmap_proto", "lz4_flex", @@ -2792,7 +2851,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12c697483ad894a8184d0fd61848e057f86b16642049993b3e6a80c959dbc90a" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "async-stream", "base64 0.13.1", "chrono", @@ -2812,7 +2871,7 @@ dependencies = [ name = "jmap_proto" version = "0.1.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "fast-float", "mail-parser", "serde", @@ -2823,15 +2882,6 @@ dependencies = [ "utils", ] -[[package]] -name = "jobserver" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" -dependencies = [ - "libc", -] - [[package]] name = "js-sys" version = "0.3.68" @@ -2847,7 +2897,7 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" dependencies = [ - "indexmap 2.2.3", + "indexmap 2.2.5", ] [[package]] @@ -2872,31 +2922,33 @@ dependencies = [ [[package]] name = "lalrpop" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da4081d44f4611b66c6dd725e6de3169f9f63905421e8626fcb86b6a898998b8" +checksum = "55cb077ad656299f160924eb2912aa147d7339ea7d69e1b5517326fdcec3c1ca" dependencies = [ "ascii-canvas", "bit-set", - "diff", "ena", - "is-terminal", - "itertools 0.10.5", + "itertools 0.11.0", "lalrpop-util", "petgraph", "regex", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", "string_cache", "term", "tiny-keccak", "unicode-xid", + "walkdir", ] [[package]] name = "lalrpop-util" -version = "0.20.0" +version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f35c735096c0293d313e8f2a641627472b83d01b937177fe76e5e2708d31e0d" +checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" +dependencies = [ + "regex-automata 0.4.5", +] [[package]] name = "lazy_static" @@ -2947,7 +2999,7 @@ dependencies = [ "tokio-stream", "tokio-util", "url", - "x509-parser", + "x509-parser 0.15.1", ] [[package]] @@ -3001,9 +3053,9 @@ dependencies = [ [[package]] name = "libsqlite3-sys" -version = "0.27.0" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" dependencies = [ "cc", "pkg-config", @@ -3045,15 +3097,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lru" -version = "0.12.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db2c024b41519440580066ba82aab04092b333e09066a5eb86c7c4890df31f22" +checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" dependencies = [ "hashbrown 0.14.3", ] @@ -3088,11 +3140,11 @@ dependencies = [ [[package]] name = "mail-auth" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224aa436a40caeef3bd3fa1b5b619b28b26d83fcc088c008536886f74ad27951" +checksum = "6bf34b7dfe585e703b74f7d9c113eb662422a74552d231222642e26d953a5d18" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "flate2", "hickory-resolver", "lru-cache", @@ -3100,8 +3152,8 @@ dependencies = [ "mail-parser", "parking_lot", "quick-xml 0.31.0", - "ring 0.17.7", - "rustls-pemfile 2.0.0", + "ring 0.17.8", + "rustls-pemfile 2.1.1", "serde", "serde_json", "zip", @@ -3164,7 +3216,7 @@ dependencies = [ name = "managesieve" version = "0.6.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "bincode", "directory", "imap", @@ -3176,7 +3228,7 @@ dependencies = [ "md5", "parking_lot", "rustls 0.22.2", - "rustls-pemfile 2.0.0", + "rustls-pemfile 2.1.1", "sieve-rs", "store", "tokio", @@ -3214,13 +3266,13 @@ checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" [[package]] name = "maybe-async" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc95a651c82daf7004c824405aa1019723644950d488571bd718e3ed84646ed" +checksum = "5cf92c10c7e361d6b99666ec1c6f9805b0bea2c3bd8c78dc6fe98ac5bd78db11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -3320,14 +3372,14 @@ version = "0.30.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56b0d8a0db9bf6d2213e11f2c701cb91387b0614361625ab7b9743b41aa4938f" dependencies = [ - "darling 0.20.5", + "darling 0.20.8", "heck", "num-bigint", "proc-macro-crate 1.3.1", "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "termcolor", "thiserror", ] @@ -3358,7 +3410,7 @@ dependencies = [ "rustls-pemfile 1.0.4", "serde", "serde_json", - "socket2 0.5.5", + "socket2 0.5.6", "thiserror", "tokio", "tokio-rustls 0.24.1", @@ -3431,7 +3483,7 @@ dependencies = [ name = "nlp" version = "0.6.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "bincode", "farmhash", "jieba-rs", @@ -3571,7 +3623,16 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" dependencies = [ - "asn1-rs", + "asn1-rs 0.5.2", +] + +[[package]] +name = "oid-registry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c958dd45046245b9c3c2547369bb634eb461670b2e7e0de552905801a648d1d" +dependencies = [ + "asn1-rs 0.6.1", ] [[package]] @@ -3582,15 +3643,15 @@ checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "openssl" -version = "0.10.63" +version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15c9d69dd87a29568d4d017cfe8ec518706046a05184e5aea92d0af890b803c8" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ "bitflags 2.4.2", "cfg-if", @@ -3609,7 +3670,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -3629,9 +3690,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.99" +version = "0.9.101" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e1bf214306098e4832460f797824c05d25aacdf896f64a985fb0fd992454ae" +checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" dependencies = [ "cc", "libc", @@ -3648,7 +3709,7 @@ checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ "futures-core", "futures-sink", - "indexmap 2.2.3", + "indexmap 2.2.5", "js-sys", "once_cell", "pin-project-lite", @@ -3815,6 +3876,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + [[package]] name = "pbkdf2" version = "0.11.0" @@ -3877,7 +3944,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.2.3", + "indexmap 2.2.5", ] [[package]] @@ -3920,7 +3987,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -3958,7 +4025,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -3996,9 +4063,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "platforms" @@ -4078,7 +4145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -4373,9 +4440,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7237101a77a10773db45d62004a272517633fbcc3df19d96455ede1122e051" +checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" dependencies = [ "either", "rayon-core", @@ -4398,7 +4465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48406db8ac1f3cbc7dcdb56ec355343817958a356ff430259bb07baf7607e1e1" dependencies = [ "pem", - "ring 0.17.7", + "ring 0.17.8", "time", "yasna", ] @@ -4492,12 +4559,6 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" -[[package]] -name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - [[package]] name = "regex-syntax" version = "0.8.2" @@ -4568,12 +4629,6 @@ dependencies = [ "quick-error", ] -[[package]] -name = "retain_mut" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086" - [[package]] name = "rfc6979" version = "0.4.0" @@ -4601,16 +4656,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4653,13 +4709,12 @@ dependencies = [ [[package]] name = "roaring" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6106b5cf8587f5834158895e9715a3c6c9716c8aefab57f1f7680917191c7873" +checksum = "a1c77081a55300e016cb86f2864415b7518741879db925b8d488a0ee0d2da6bf" dependencies = [ "bytemuck", "byteorder", - "retain_mut", ] [[package]] @@ -4715,9 +4770,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.30.0" +version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" dependencies = [ "bitflags 2.4.2", "fallible-iterator 0.3.0", @@ -4822,7 +4877,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver 1.0.21", + "semver 1.0.22", ] [[package]] @@ -4866,7 +4921,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-webpki 0.101.7", "sct", ] @@ -4878,7 +4933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-pki-types", "rustls-webpki 0.102.2", "subtle", @@ -4908,9 +4963,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" +checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab" dependencies = [ "base64 0.21.7", "rustls-pki-types", @@ -4918,9 +4973,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a716eb65e3158e90e17cd93d855216e27bde02745ab842f2cab4a39dba1bacf" +checksum = "5ede67b28608b4c60685c7d54122d4400d90f62b40caee7700e700380a390fa8" [[package]] name = "rustls-webpki" @@ -4938,7 +4993,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -4948,7 +5003,7 @@ version = "0.102.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "rustls-pki-types", "untrusted 0.9.0", ] @@ -4961,9 +5016,9 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" [[package]] name = "ryu" -version = "1.0.16" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" [[package]] name = "salsa20" @@ -4974,6 +5029,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "saturating" version = "0.1.0" @@ -5022,7 +5086,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -5080,9 +5144,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97ed7a9823b74f99c7742f5336af7be5ecd3eeafcb1507d1fa93347b1d589b0" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" [[package]] name = "semver-parser" @@ -5092,9 +5156,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "sequoia-openpgp" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26e9c71323d9848404e343a6b5c3a73de10bc496ca3481b66586ba9064de027e" +checksum = "ebf154ce4af3d7983de8fded403f98ff9eb3ee38dffccea0472ac38aa4276df4" dependencies = [ "aes", "aes-gcm", @@ -5146,9 +5210,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" dependencies = [ "serde_derive", ] @@ -5164,20 +5228,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.196" +version = "1.0.197" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] name = "serde_json" -version = "1.0.113" +version = "1.0.114" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" dependencies = [ "itoa", "ryu", @@ -5240,7 +5304,7 @@ checksum = "91d129178576168c589c9ec973feedf7d3126c01ac2bf08795109aa35b69fb8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -5335,7 +5399,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25494c13da6c336430906aa783e4bb2ae251c84158d6e5a4fdf0449a779c2521" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "bincode", "fancy-regex", "mail-builder", @@ -5400,7 +5464,7 @@ checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" name = "smtp" version = "0.6.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "bincode", "blake3", "dashmap", @@ -5408,7 +5472,7 @@ dependencies = [ "directory", "form_urlencoded", "http-body-util", - "hyper 1.1.0", + "hyper 1.2.0", "hyper-util", "idna 0.5.0", "imagesize", @@ -5428,7 +5492,7 @@ dependencies = [ "regex", "reqwest", "rustls 0.22.2", - "rustls-pemfile 2.0.0", + "rustls-pemfile 2.1.1", "rustls-pki-types", "serde", "serde_json", @@ -5444,7 +5508,7 @@ dependencies = [ "utils", "webpki-roots 0.26.1", "whatlang", - "x509-parser", + "x509-parser 0.16.0", ] [[package]] @@ -5491,12 +5555,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5533,6 +5597,7 @@ dependencies = [ "human-size", "indicatif", "jmap-client", + "mail-auth", "mail-parser", "num_cpus", "prettytable-rs", @@ -5575,7 +5640,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" name = "store" version = "0.1.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "arc-swap", "async-trait", "bincode", @@ -5601,7 +5666,7 @@ dependencies = [ "redis", "regex", "reqwest", - "ring 0.17.7", + "ring 0.17.8", "roaring", "rocksdb", "rusqlite", @@ -5683,9 +5748,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" dependencies = [ "proc-macro2", "quote", @@ -5701,7 +5766,7 @@ dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -5722,6 +5787,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "system-configuration" version = "0.5.1" @@ -5762,9 +5838,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.0" +version = "3.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", @@ -5796,7 +5872,7 @@ dependencies = [ name = "tests" version = "0.1.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "async-trait", "base64 0.21.7", "bytes", @@ -5808,7 +5884,7 @@ dependencies = [ "flate2", "futures", "http-body-util", - "hyper 1.1.0", + "hyper 1.2.0", "hyper-util", "imap", "imap_proto", @@ -5825,7 +5901,7 @@ dependencies = [ "rayon", "reqwest", "rustls 0.22.2", - "rustls-pemfile 2.0.0", + "rustls-pemfile 2.1.1", "rustls-pki-types", "serde", "serde_json", @@ -5858,14 +5934,14 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -5950,7 +6026,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.5", + "socket2 0.5.6", "tokio-macros", "windows-sys 0.48.0", ] @@ -5973,7 +6049,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -5996,7 +6072,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand", - "socket2 0.5.5", + "socket2 0.5.6", "tokio", "tokio-util", "whoami", @@ -6076,7 +6152,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.3", + "indexmap 2.2.5", "toml_datetime", "winnow", ] @@ -6087,7 +6163,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" dependencies = [ - "indexmap 2.2.3", + "indexmap 2.2.5", "toml_datetime", "winnow", ] @@ -6183,7 +6259,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -6351,18 +6427,18 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] [[package]] name = "unicode-script" -version = "0.5.5" +version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d817255e1bed6dfd4ca47258685d14d2bdcfbc64fdc9e3819bd5848057b8ecc" +checksum = "ad8d71f5726e5f285a935e9fe8edfd53f0491eb6e9a5774097fdabee7cd8c9cd" [[package]] name = "unicode-security" @@ -6441,7 +6517,7 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" name = "utils" version = "0.6.0" dependencies = [ - "ahash 0.8.8", + "ahash 0.8.10", "arc-swap", "base64 0.21.7", "blake3", @@ -6462,9 +6538,9 @@ dependencies = [ "rcgen", "regex", "reqwest", - "ring 0.17.7", + "ring 0.17.8", "rustls 0.22.2", - "rustls-pemfile 2.0.0", + "rustls-pemfile 2.1.1", "rustls-pki-types", "serde", "serde_json", @@ -6477,7 +6553,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "webpki-roots 0.26.1", - "x509-parser", + "x509-parser 0.16.0", ] [[package]] @@ -6513,6 +6589,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "walkdir" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -6549,7 +6635,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "wasm-bindgen-shared", ] @@ -6583,7 +6669,7 @@ checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -6633,7 +6719,7 @@ version = "0.22.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -6745,7 +6831,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -6763,7 +6849,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.4", ] [[package]] @@ -6783,17 +6869,17 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.4", + "windows_aarch64_msvc 0.52.4", + "windows_i686_gnu 0.52.4", + "windows_i686_msvc 0.52.4", + "windows_x86_64_gnu 0.52.4", + "windows_x86_64_gnullvm 0.52.4", + "windows_x86_64_msvc 0.52.4", ] [[package]] @@ -6804,9 +6890,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" [[package]] name = "windows_aarch64_msvc" @@ -6816,9 +6902,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" [[package]] name = "windows_i686_gnu" @@ -6828,9 +6914,9 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" [[package]] name = "windows_i686_msvc" @@ -6840,9 +6926,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" [[package]] name = "windows_x86_64_gnu" @@ -6852,9 +6938,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" [[package]] name = "windows_x86_64_gnullvm" @@ -6864,9 +6950,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" [[package]] name = "windows_x86_64_msvc" @@ -6876,9 +6962,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" [[package]] name = "winnow" @@ -6925,12 +7011,29 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" dependencies = [ - "asn1-rs", + "asn1-rs 0.5.2", "data-encoding", - "der-parser", + "der-parser 8.2.0", "lazy_static", "nom", - "oid-registry", + "oid-registry 0.6.1", + "rusticata-macros", + "thiserror", + "time", +] + +[[package]] +name = "x509-parser" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +dependencies = [ + "asn1-rs 0.6.1", + "data-encoding", + "der-parser 9.0.0", + "lazy_static", + "nom", + "oid-registry 0.7.0", "rusticata-macros", "thiserror", "time", @@ -6955,9 +7058,9 @@ checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" [[package]] name = "xxhash-rust" -version = "0.8.8" +version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53be06678ed9e83edb1745eb72efc0bbcd7b5c3c35711a860906aed827a13d61" +checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" [[package]] name = "yasna" @@ -6985,7 +7088,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] @@ -7005,7 +7108,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.52", ] [[package]] diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 7b46d35f..a10e5849 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -29,3 +29,4 @@ human-size = "0.4.2" futures = "0.3.28" pwhash = "1.0.0" rand = "0.8.5" +mail-auth = "0.3.7" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 43f292a9..dee44394 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -208,6 +208,20 @@ impl Client { url: &str, body: Option, ) -> R { + self.try_http_request(method, url, body) + .await + .unwrap_or_else(|| { + eprintln!("Request failed: No data returned."); + std::process::exit(1); + }) + } + + pub async fn try_http_request( + &self, + method: Method, + url: &str, + body: Option, + ) -> Option { let url = format!( "{}{}{}", self.url, @@ -240,6 +254,9 @@ impl Client { match response.status() { StatusCode::OK => (), + StatusCode::NOT_FOUND => { + return None; + } StatusCode::UNAUTHORIZED => { eprintln!("Authentication failed. Make sure the credentials are correct and that the account has administrator rights."); std::process::exit(1); @@ -258,7 +275,7 @@ impl Client { ) .unwrap_result("deserialize response") { - Response::Data { data } => data, + Response::Data { data } => Some(data), Response::Error { error, details } => { eprintln!("Request failed: {details} ({error:?})"); std::process::exit(1); diff --git a/crates/cli/src/modules/mod.rs b/crates/cli/src/modules/mod.rs index b4f22d9d..835f6af9 100644 --- a/crates/cli/src/modules/mod.rs +++ b/crates/cli/src/modules/mod.rs @@ -117,6 +117,12 @@ pub enum PrincipalField { Members, } +#[derive(Clone, serde::Serialize, serde::Deserialize, Default)] +pub struct List { + pub items: Vec, + pub total: u64, +} + #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct PrincipalUpdate { action: PrincipalAction, diff --git a/crates/cli/src/modules/queue.rs b/crates/cli/src/modules/queue.rs index 6c923e19..8bac4887 100644 --- a/crates/cli/src/modules/queue.rs +++ b/crates/cli/src/modules/queue.rs @@ -21,7 +21,10 @@ * for more details. */ -use super::cli::{Client, QueueCommands}; +use super::{ + cli::{Client, QueueCommands}, + List, +}; use console::Term; use human_size::{Byte, SpecificSize}; use mail_parser::DateTime; @@ -99,65 +102,62 @@ impl QueueCommands { .map(|p| Cell::new(p).with_style(Attr::Bold)) .collect(), )); - for (message, id) in client - .http_request::>, String>( - Method::GET, - &build_query("/api/queue/status?ids=", chunk), - None, - ) - .await - .into_iter() - .zip(chunk) - { - if let Some(message) = message { - let mut rcpts = String::new(); - let mut deliver_at = i64::MAX; - let mut deliver_pos = 0; - for (pos, domain) in message.domains.iter().enumerate() { - if let Some(next_retry) = &domain.next_retry { - let ts = next_retry.to_timestamp(); - if ts < deliver_at { - deliver_at = ts; - deliver_pos = pos; - } - } - for rcpt in &domain.recipients { - if !rcpts.is_empty() { - rcpts.push('\n'); - } - rcpts.push_str(&rcpt.address); - rcpts.push_str(" ("); - rcpts.push_str(rcpt.status.status_short()); - rcpts.push(')'); + for id in chunk { + let message = client + .http_request::( + Method::GET, + &format!("/api/queue/messages/{id}"), + None, + ) + .await; + + let mut rcpts = String::new(); + let mut deliver_at = i64::MAX; + let mut deliver_pos = 0; + for (pos, domain) in message.domains.iter().enumerate() { + if let Some(next_retry) = &domain.next_retry { + let ts = next_retry.to_timestamp(); + if ts < deliver_at { + deliver_at = ts; + deliver_pos = pos; } } - - let mut cells = Vec::new(); - cells.push(Cell::new(&format!("{id:X}"))); - cells.push(if deliver_at != i64::MAX { - Cell::new( - &message.domains[deliver_pos] - .next_retry - .as_ref() - .unwrap() - .to_rfc822(), - ) - } else { - Cell::new("None") - }); - cells.push(Cell::new(if !message.return_path.is_empty() { - &message.return_path - } else { - "<>" - })); - cells.push(Cell::new(&rcpts)); - cells.push(Cell::new( - &SpecificSize::new(message.size as u32, Byte) - .unwrap() - .to_string(), - )); - table.add_row(Row::new(cells)); + for rcpt in &domain.recipients { + if !rcpts.is_empty() { + rcpts.push('\n'); + } + rcpts.push_str(&rcpt.address); + rcpts.push_str(" ("); + rcpts.push_str(rcpt.status.status_short()); + rcpts.push(')'); + } } + + let mut cells = Vec::new(); + cells.push(Cell::new(&format!("{id:X}"))); + cells.push(if deliver_at != i64::MAX { + Cell::new( + &message.domains[deliver_pos] + .next_retry + .as_ref() + .unwrap() + .to_rfc822(), + ) + } else { + Cell::new("None") + }); + cells.push(Cell::new(if !message.return_path.is_empty() { + &message.return_path + } else { + "<>" + })); + cells.push(Cell::new(&rcpts)); + cells.push(Cell::new( + &SpecificSize::new(message.size as u32, Byte) + .unwrap() + .to_string(), + )); + table.add_row(Row::new(cells)); } eprintln!(); @@ -173,21 +173,20 @@ impl QueueCommands { eprintln!("\n{ids_len} queued message(s) found.") } QueueCommands::Status { ids } => { - for (message, id) in client - .http_request::>, String>( - Method::GET, - &build_query("/api/queue/status?ids=", &parse_ids(&ids)), - None, - ) - .await - .into_iter() - .zip(&ids) - { + for (uid, id) in parse_ids(&ids).into_iter().zip(ids) { + let message = client + .try_http_request::( + Method::GET, + &format!("/api/queue/messages/{uid}"), + None, + ) + .await; let mut table = Table::new(); table.add_row(Row::new(vec![ Cell::new("ID").with_style(Attr::Bold), - Cell::new(id), + Cell::new(&id), ])); + if let Some(message) = message { table.add_row(Row::new(vec![ Cell::new("Sender").with_style(Attr::Bold), @@ -316,30 +315,31 @@ impl QueueCommands { std::process::exit(1); } - let mut query = form_urlencoded::Serializer::new("/api/queue/retry?".to_string()); - - if let Some(filter) = &domain { - query.append_pair("filter", filter); - } - if let Some(at) = time { - query.append_pair("at", &at.to_rfc3339()); - } - query.append_pair("ids", &append_ids(String::new(), &parsed_ids)); - let mut success_count = 0; let mut failed_list = vec![]; - for (success, id) in client - .http_request::, String>(Method::GET, &query.finish(), None) - .await - .into_iter() - .zip(ids) - { - if success { + + for id in parsed_ids { + let mut query = + form_urlencoded::Serializer::new(format!("/api/queue/messages/{id}")); + + if let Some(filter) = &domain { + query.append_pair("filter", filter); + } + if let Some(at) = time { + query.append_pair("at", &at.to_rfc3339()); + } + + if client + .try_http_request::(Method::PATCH, &query.finish(), None) + .await + .unwrap_or(false) + { success_count += 1; } else { - failed_list.push(id); + failed_list.push(id.to_string()); } } + eprint!("\nSuccessfully rescheduled {success_count} message(s)."); if !failed_list.is_empty() { eprint!(" Unable to reschedule id(s): {}.", failed_list.join(", ")); @@ -371,27 +371,28 @@ impl QueueCommands { std::process::exit(1); } - let mut query = form_urlencoded::Serializer::new("/api/queue/cancel?".to_string()); - - if let Some(filter) = &rcpt { - query.append_pair("filter", filter); - } - query.append_pair("ids", &append_ids(String::new(), &parsed_ids)); - let mut success_count = 0; let mut failed_list = vec![]; - for (success, id) in client - .http_request::, String>(Method::GET, &query.finish(), None) - .await - .into_iter() - .zip(ids) - { - if success { + + for id in parsed_ids { + let mut query = + form_urlencoded::Serializer::new(format!("/api/queue/messages/{id}")); + + if let Some(filter) = &rcpt { + query.append_pair("filter", filter); + } + + if client + .try_http_request::(Method::DELETE, &query.finish(), None) + .await + .unwrap_or(false) + { success_count += 1; } else { - failed_list.push(id); + failed_list.push(id.to_string()); } } + eprint!("\nCancelled delivery of {success_count} message(s)."); if !failed_list.is_empty() { eprint!( @@ -413,7 +414,7 @@ impl Client { before: &Option, after: &Option, ) -> Vec { - let mut query = form_urlencoded::Serializer::new("/api/queue/list?".to_string()); + let mut query = form_urlencoded::Serializer::new("/api/queue/messages".to_string()); if let Some(sender) = from { query.append_pair("from", sender); @@ -428,8 +429,9 @@ impl Client { query.append_pair("after", &after.to_rfc3339()); } - self.http_request::, String>(Method::GET, &query.finish(), None) + self.http_request::, String>(Method::GET, &query.finish(), None) .await + .items } } @@ -479,22 +481,6 @@ fn parse_ids(ids: &[String]) -> Vec { result } -fn build_query(path: &str, ids: &[u64]) -> String { - let mut query = String::with_capacity(path.len() + (ids.len() * 10)); - query.push_str(path); - append_ids(query, ids) -} - -fn append_ids(mut query: String, ids: &[u64]) -> String { - for (pos, id) in ids.iter().enumerate() { - if pos != 0 { - query.push(','); - } - query.push_str(&id.to_string()); - } - query -} - impl Status { fn status_short(&self) -> &str { match self { diff --git a/crates/cli/src/modules/report.rs b/crates/cli/src/modules/report.rs index 60f72d63..ef019ffb 100644 --- a/crates/cli/src/modules/report.rs +++ b/crates/cli/src/modules/report.rs @@ -22,24 +22,83 @@ */ use super::cli::{Client, ReportCommands, ReportFormat}; -use crate::modules::queue::deserialize_datetime; +use crate::modules::{queue::deserialize_datetime, List}; use console::Term; use human_size::{Byte, SpecificSize}; +use mail_auth::{ + dmarc::URI, + mta_sts::ReportUri, + report::{self, tlsrpt::TlsReport}, +}; use mail_parser::DateTime; -use prettytable::{format::Alignment, Attr, Cell, Row, Table}; +use prettytable::{format, Attr, Cell, Row, Table}; use reqwest::Method; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Deserialize)] -pub struct Report { - pub domain: String, - #[serde(rename = "type")] - pub type_: ReportFormat, - #[serde(deserialize_with = "deserialize_datetime")] - pub range_from: DateTime, - #[serde(deserialize_with = "deserialize_datetime")] - pub range_to: DateTime, - pub size: usize, +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum Report { + Tls { + id: String, + domain: String, + #[serde(deserialize_with = "deserialize_datetime")] + range_from: DateTime, + #[serde(deserialize_with = "deserialize_datetime")] + range_to: DateTime, + report: TlsReport, + rua: Vec, + }, + Dmarc { + id: String, + domain: String, + #[serde(deserialize_with = "deserialize_datetime")] + range_from: DateTime, + #[serde(deserialize_with = "deserialize_datetime")] + range_to: DateTime, + report: report::Report, + rua: Vec, + }, +} + +impl Report { + pub fn domain(&self) -> &str { + match self { + Report::Tls { domain, .. } => domain, + Report::Dmarc { domain, .. } => domain, + } + } + + pub fn type_(&self) -> &str { + match self { + Report::Tls { .. } => "TLS", + Report::Dmarc { .. } => "DMARC", + } + } + + pub fn range_from(&self) -> &DateTime { + match self { + Report::Tls { range_from, .. } => range_from, + Report::Dmarc { range_from, .. } => range_from, + } + } + + pub fn range_to(&self) -> &DateTime { + match self { + Report::Tls { range_to, .. } => range_to, + Report::Dmarc { range_to, .. } => range_to, + } + } + + pub fn num_records(&self) -> usize { + match self { + Report::Tls { report, .. } => report + .policies + .iter() + .map(|p| p.failure_details.len()) + .sum(), + Report::Dmarc { report, .. } => report.records().len(), + } + } } impl ReportCommands { @@ -51,7 +110,7 @@ impl ReportCommands { page_size, } => { let stdout = Term::buffered_stdout(); - let mut query = form_urlencoded::Serializer::new("/api/report/list?".to_string()); + let mut query = form_urlencoded::Serializer::new("/api/queue/reports".to_string()); if let Some(domain) = &domain { query.append_pair("domain", domain); @@ -61,8 +120,9 @@ impl ReportCommands { } let ids = client - .http_request::, String>(Method::GET, &query.finish(), None) - .await; + .http_request::, String>(Method::GET, &query.finish(), None) + .await + .items; let ids_len = ids.len(); let page_size = page_size.map(|p| std::cmp::max(p, 1)).unwrap_or(20); let pages_total = (ids_len as f64 / page_size as f64).ceil() as usize; @@ -70,30 +130,29 @@ impl ReportCommands { // Build table let mut table = Table::new(); table.add_row(Row::new( - ["ID", "Domain", "Type", "From Date", "To Date", "Size"] + ["ID", "Domain", "Type", "From Date", "To Date", "Records"] .iter() .map(|p| Cell::new(p).with_style(Attr::Bold)) .collect(), )); - for (report, id) in client - .http_request::>, String>( - Method::GET, - &format!("/api/report/status?ids={}", chunk.join(",")), - None, - ) - .await - .into_iter() - .zip(chunk) - { + for id in chunk { + let report = client + .try_http_request::( + Method::GET, + &format!("/api/queue/reports/{id}"), + None, + ) + .await; + if let Some(report) = report { table.add_row(Row::new(vec![ Cell::new(id), - Cell::new(&report.domain), - Cell::new(report.type_.name()), - Cell::new(&report.range_from.to_rfc822()), - Cell::new(&report.range_to.to_rfc822()), + Cell::new(report.domain()), + Cell::new(report.type_()), + Cell::new(&report.range_from().to_rfc822()), + Cell::new(&report.range_to().to_rfc822()), Cell::new( - &SpecificSize::new(report.size as u32, Byte) + &SpecificSize::new(report.num_records() as u32, Byte) .unwrap() .to_string(), ), @@ -114,42 +173,41 @@ impl ReportCommands { eprintln!("\n{ids_len} queued message(s) found.") } ReportCommands::Status { ids } => { - for (report, id) in client - .http_request::>, String>( - Method::GET, - &format!("/api/report/status?ids={}", ids.join(",")), - None, - ) - .await - .into_iter() - .zip(&ids) - { + for id in ids { + let report = client + .try_http_request::( + Method::GET, + &format!("/api/queue/reports/{id}"), + None, + ) + .await; + let mut table = Table::new(); table.add_row(Row::new(vec![ Cell::new("ID").with_style(Attr::Bold), - Cell::new(id), + Cell::new(&id), ])); if let Some(report) = report { table.add_row(Row::new(vec![ Cell::new("Domain Name").with_style(Attr::Bold), - Cell::new(&report.domain), + Cell::new(report.domain()), ])); table.add_row(Row::new(vec![ Cell::new("Type").with_style(Attr::Bold), - Cell::new(report.type_.name()), + Cell::new(report.type_()), ])); table.add_row(Row::new(vec![ Cell::new("From Date").with_style(Attr::Bold), - Cell::new(&report.range_from.to_rfc822()), + Cell::new(&report.range_from().to_rfc822()), ])); table.add_row(Row::new(vec![ Cell::new("To Date").with_style(Attr::Bold), - Cell::new(&report.range_to.to_rfc822()), + Cell::new(&report.range_to().to_rfc822()), ])); table.add_row(Row::new(vec![ - Cell::new("Size").with_style(Attr::Bold), + Cell::new("Records").with_style(Attr::Bold), Cell::new( - &SpecificSize::new(report.size as u32, Byte) + &SpecificSize::new(report.num_records() as u32, Byte) .unwrap() .to_string(), ), @@ -157,7 +215,7 @@ impl ReportCommands { } else { table.add_row(Row::new(vec![Cell::new_align( "-- Not found --", - Alignment::CENTER, + format::Alignment::CENTER, ) .with_hspan(2)])); } @@ -170,17 +228,16 @@ impl ReportCommands { ReportCommands::Cancel { ids } => { let mut success_count = 0; let mut failed_list = vec![]; - for (success, id) in client - .http_request::, String>( - Method::GET, - &format!("/api/report/cancel?ids={}", ids.join(",")), - None, - ) - .await - .into_iter() - .zip(ids) - { - if success { + for id in ids { + let success = client + .try_http_request::( + Method::DELETE, + &format!("/api/queue/reports/{id}"), + None, + ) + .await; + + if success.unwrap_or_default() { success_count += 1; } else { failed_list.push(id); @@ -206,11 +263,4 @@ impl ReportFormat { ReportFormat::Tls => "tls", } } - - fn name(&self) -> &'static str { - match self { - ReportFormat::Dmarc => "DMARC", - ReportFormat::Tls => "TLS", - } - } } diff --git a/crates/jmap/src/api/admin.rs b/crates/jmap/src/api/admin.rs index 6b9679fd..e3270ce0 100644 --- a/crates/jmap/src/api/admin.rs +++ b/crates/jmap/src/api/admin.rs @@ -346,7 +346,7 @@ impl JMAP { } ("store", Some("maintenance"), &Method::GET) => { match self.store.purge_blobs(self.blob_store.clone()).await { - Ok(_) => match self.store.purge_bitmaps().await { + Ok(_) => match self.store.purge_store().await { Ok(_) => JsonResponse::new(json!({ "data": (), })) @@ -456,9 +456,9 @@ impl JMAP { } } ("oauth", _, _) => self.handle_api_request(req, body, access_token).await, - (path_1 @ ("queue" | "report"), Some(path_2), &Method::GET) => { + (path_1 @ ("queue" | "reports"), Some(path_2), &Method::GET) => { self.smtp - .handle_manage_request(req.uri(), req.method(), path_1, path_2) + .handle_manage_request(req.uri(), req.method(), path_1, path_2, path.next()) .await } _ => RequestError::not_found().into_http_response(), diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index f214910d..24305eeb 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -44,7 +44,7 @@ dashmap = "5.4" blake3 = "1.3" lru-cache = "0.1.2" rand = "0.8.5" -x509-parser = "0.15.0" +x509-parser = "0.16.0" reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots", "blocking"] } serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0" @@ -53,7 +53,7 @@ lazy_static = "1.4" whatlang = "0.16" imagesize = "0.12" idna = "0.5" -decancer = "1.6.1" +decancer = "3.0.1" unicode-security = "0.1.0" infer = "0.15.0" bincode = "1.3.1" diff --git a/crates/smtp/src/config/mod.rs b/crates/smtp/src/config/mod.rs index 8ce29c2e..cd4a84c6 100644 --- a/crates/smtp/src/config/mod.rs +++ b/crates/smtp/src/config/mod.rs @@ -30,12 +30,7 @@ pub mod session; pub mod shared; pub mod throttle; -use std::{ - net::SocketAddr, - path::PathBuf, - sync::{atomic::AtomicU64, Arc}, - time::Duration, -}; +use std::{net::SocketAddr, sync::Arc, time::Duration}; use ahash::AHashMap; use directory::Directories; @@ -49,6 +44,7 @@ use store::Stores; use utils::{ config::{if_block::IfBlock, utils::ConstantValue, Rate, Server, ServerProtocol}, expr::{Expression, Token}, + snowflake::SnowflakeIdGenerator, }; use crate::{ @@ -250,8 +246,8 @@ pub struct ReportConfig { pub struct ReportAnalysis { pub addresses: Vec, pub forward: bool, - pub store: Option, - pub report_id: AtomicU64, + pub store: Option, + pub report_id: SnowflakeIdGenerator, } pub enum AddressMatch { diff --git a/crates/smtp/src/config/report.rs b/crates/smtp/src/config/report.rs index cc858260..930f5efa 100644 --- a/crates/smtp/src/config/report.rs +++ b/crates/smtp/src/config/report.rs @@ -31,6 +31,7 @@ use utils::{ Config, }, expr::{Constant, Variable}, + snowflake::SnowflakeIdGenerator, }; use super::{ @@ -98,7 +99,10 @@ impl ConfigReport for Config { addresses, forward: self.property("report.analysis.forward")?.unwrap_or(false), store: self.property("report.analysis.store")?, - report_id: 0.into(), + report_id: self + .property::("storage.cluster.node-id")? + .map(SnowflakeIdGenerator::with_node_id) + .unwrap_or_else(SnowflakeIdGenerator::new), }, }) } diff --git a/crates/smtp/src/core/management.rs b/crates/smtp/src/core/management.rs index e7ed4ff6..cc62ac88 100644 --- a/crates/smtp/src/core/management.rs +++ b/crates/smtp/src/core/management.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{borrow::Cow, net::IpAddr, sync::Arc}; +use std::{borrow::Cow, collections::HashMap, net::IpAddr, str::FromStr, sync::Arc}; use directory::{AuthResult, Type}; use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full}; @@ -33,20 +33,32 @@ use hyper::{ Method, StatusCode, Uri, }; use hyper_util::rt::TokioIo; +use mail_auth::{ + dmarc::URI, + mta_sts::ReportUri, + report::{ + self, + tlsrpt::{FailureDetails, Policy, TlsReport}, + Feedback, + }, +}; use mail_parser::{decoders::base64::base64_decode, DateTime}; use mail_send::Credentials; use serde::{Deserializer, Serializer}; use serde_json::json; use store::{ - write::{key::DeserializeBigEndian, now, Bincode, QueueClass, ReportEvent, ValueClass}, - Deserialize, IterateParams, ValueKey, + write::{ + key::DeserializeBigEndian, now, BatchBuilder, Bincode, QueueClass, ReportClass, + ReportEvent, ValueClass, + }, + Deserialize, IterateParams, ValueKey, U64_LEN, }; use utils::listener::{limiter::InFlight, SessionData, SessionManager, SessionStream}; use crate::{ queue::{self, ErrorDetails, HostResponse, QueueId, Status}, - reporting::{dmarc::DmarcFormat, tls::TlsFormat}, + reporting::analysis::IncomingReport, }; use super::{SmtpAdminSessionManager, SMTP}; @@ -110,7 +122,8 @@ pub enum Report { #[serde(deserialize_with = "deserialize_datetime")] #[serde(serialize_with = "serialize_datetime")] range_to: DateTime, - report: TlsFormat, + report: TlsReport, + rua: Vec, }, Dmarc { id: String, @@ -121,7 +134,8 @@ pub enum Report { #[serde(deserialize_with = "deserialize_datetime")] #[serde(serialize_with = "serialize_datetime")] range_to: DateTime, - report: DmarcFormat, + report: report::Report, + rua: Vec, }, } @@ -318,6 +332,7 @@ impl SMTP { req.method(), path.next().unwrap_or_default(), path.next().unwrap_or_default(), + path.next(), ) .await) } @@ -328,484 +343,293 @@ impl SMTP { method: &Method, path_1: &str, path_2: &str, + path_3: Option<&str>, ) -> hyper::Response> { - let (status, response) = match (method, path_1, path_2) { - (&Method::GET, "queue", "list") => { - let mut text = None; - let mut from = None; - let mut to = None; - let mut before = None; - let mut after = None; - let mut error = None; - let mut page: usize = 0; - let mut limit: usize = 0; - let mut values = false; + let params = UrlParams::new(uri); - if let Some(query) = uri.query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "text" => { - if !value.is_empty() { - text = value.into_owned().into(); - } - } - "from" => { - if !value.is_empty() { - from = value.into_owned().into(); - } - } - "to" => { - if !value.is_empty() { - to = value.into_owned().into(); - } - } - "after" => match value.parse_timestamp() { - Ok(dt) => { - after = dt.into(); - } - Err(reason) => { - error = reason.into(); - break; - } - }, - "before" => match value.parse_timestamp() { - Ok(dt) => { - before = dt.into(); - } - Err(reason) => { - error = reason.into(); - break; - } - }, - "values" => { - values = true; - } - "limit" => { - limit = value.parse().unwrap_or_default(); - } - "page" => { - page = value.parse().unwrap_or_default(); - } - _ => { - error = format!("Invalid parameter {key:?}.").into(); - break; - } - } - } - } + let (status, response) = match (method, path_1, path_2, path_3) { + (&Method::GET, "queue", "messages", None) => { + let text = params.get("text"); + let from = params.get("from"); + let to = params.get("to"); + let before = params.parse::("before").map(|t| t.into_inner()); + let after = params.parse::("after").map(|t| t.into_inner()); + let page: usize = params.parse::("page").unwrap_or_default(); + let limit: usize = params.parse::("limit").unwrap_or_default(); + let values = params.has_key("values"); - match error { - None => { - let mut result_ids = Vec::new(); - let mut result_values = Vec::new(); - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(0))); - let to_key = - ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))); - let has_filters = text.is_some() - || from.is_some() - || to.is_some() - || before.is_some() - || after.is_some(); - let mut offset = page.saturating_sub(1) * limit; - let mut total = 0; - let mut total_returned = 0; - let _ = - self.shared - .default_data_store - .iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - let message = - Bincode::::deserialize(value)?.inner; - let matches = - !has_filters - || (text - .as_ref() - .map(|text| { - message.return_path.contains(text) - || message.recipients.iter().any(|r| { - r.address_lcase.contains(text) - }) - }) - .unwrap_or_else(|| { - from.as_ref().map_or(true, |from| { - message.return_path.contains(from) - }) && to.as_ref().map_or(true, |to| { - message.recipients.iter().any(|r| { - r.address_lcase.contains(to) - }) - }) - }) - && before.as_ref().map_or(true, |before| { - message.next_delivery_event() < *before - }) - && after.as_ref().map_or(true, |after| { - message.next_delivery_event() > *after - })); + let mut result_ids = Vec::new(); + let mut result_values = Vec::new(); + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(0))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::Message(u64::MAX))); + let has_filters = text.is_some() + || from.is_some() + || to.is_some() + || before.is_some() + || after.is_some(); + let mut offset = page.saturating_sub(1) * limit; + let mut total = 0; + let mut total_returned = 0; + let _ = self + .shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + let message = Bincode::::deserialize(value)?.inner; + let matches = !has_filters + || (text + .as_ref() + .map(|text| { + message.return_path.contains(text) + || message + .recipients + .iter() + .any(|r| r.address_lcase.contains(text)) + }) + .unwrap_or_else(|| { + from.as_ref() + .map_or(true, |from| message.return_path.contains(from)) + && to.as_ref().map_or(true, |to| { + message + .recipients + .iter() + .any(|r| r.address_lcase.contains(to)) + }) + }) + && before.as_ref().map_or(true, |before| { + message.next_delivery_event() < *before + }) + && after.as_ref().map_or(true, |after| { + message.next_delivery_event() > *after + })); - if matches { - if offset == 0 { - if limit == 0 || total_returned < limit { - if values { - result_values.push(Message::from(&message)); - } else { - result_ids.push(key.deserialize_be_u64(1)?); - } - total_returned += 1; - } - } else { - offset -= 1; - } - - total += 1; - } - - Ok(true) - }, - ) - .await; - - ( - StatusCode::OK, - if values { - serde_json::to_string(&json!({ - "data": { - "items": result_values, - "total": total, - }, - })) - } else { - serde_json::to_string(&json!({ - "data": { - "items": result_ids, - "total": total, - }, - })) - } - .unwrap_or_default(), - ) - } - Some(error) => error.into_bad_request(), - } - } - (&Method::GET, "queue", "status") => { - let mut queue_ids = Vec::new(); - let mut error = None; - - if let Some(query) = uri.query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "id" | "ids" => match value.parse_queue_ids() { - Ok(ids) => { - queue_ids = ids; - } - Err(reason) => { - error = reason.into(); - break; - } - }, - _ => { - error = format!("Invalid parameter {key:?}.").into(); - break; - } - } - } - } - - match error { - None => { - let mut result = Vec::with_capacity(queue_ids.len()); - for queue_id in queue_ids { - if let Some(message) = self.read_message(queue_id).await { - result.push(Message::from(&message).into()); - } else { - result.push(None); - } - } - - ( - StatusCode::OK, - serde_json::to_string(&Response { data: result }).unwrap_or_default(), - ) - } - Some(error) => error.into_bad_request(), - } - } - (&Method::GET, "queue", "retry") => { - let mut queue_ids = Vec::new(); - let mut time = now(); - let mut item = None; - let mut error = None; - - if let Some(query) = uri.query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "id" | "ids" => match value.parse_queue_ids() { - Ok(ids) => { - queue_ids = ids; - } - Err(reason) => { - error = reason.into(); - break; - } - }, - "at" => match value.parse_timestamp() { - Ok(dt) => { - time = dt; - } - Err(reason) => { - error = reason.into(); - break; - } - }, - "filter" => { - if !value.is_empty() { - item = value.into_owned().into(); - } - } - _ => { - error = format!("Invalid parameter {key:?}.").into(); - break; - } - } - } - } - - match error { - None => { - let mut result = Vec::with_capacity(queue_ids.len()); - - for queue_id in queue_ids { - let mut found = false; - - if let Some(mut message) = self.read_message(queue_id).await { - let prev_event = message.next_event().unwrap_or_default(); - - for domain in &mut message.domains { - if matches!( - domain.status, - Status::Scheduled | Status::TemporaryFailure(_) - ) && item - .as_ref() - .map_or(true, |item| domain.domain.contains(item)) - { - domain.retry.due = time; - if domain.expires > time { - domain.expires = time + 10; - } - found = true; - } - } - - if found { - let next_event = message.next_event().unwrap_or_default(); - message - .save_changes(self, prev_event.into(), next_event.into()) - .await; - } - } - - result.push(found); - } - - if result.iter().any(|r| *r) { - let _ = self.queue.tx.send(queue::Event::Reload).await; - } - - ( - StatusCode::OK, - serde_json::to_string(&Response { data: result }).unwrap_or_default(), - ) - } - Some(error) => error.into_bad_request(), - } - } - (&Method::GET, "queue", "cancel") => { - let mut queue_ids = Vec::new(); - let mut item = None; - let mut error = None; - - if let Some(query) = uri.query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "id" | "ids" => match value.parse_queue_ids() { - Ok(ids) => { - queue_ids = ids; - } - Err(reason) => { - error = reason.into(); - break; - } - }, - "filter" => { - if !value.is_empty() { - item = value.into_owned().into(); - } - } - _ => { - error = format!("Invalid parameter {key:?}.").into(); - break; - } - } - } - } - - match error { - None => { - let mut result = Vec::with_capacity(queue_ids.len()); - - for queue_id in queue_ids { - let mut found = false; - - if let Some(mut message) = self.read_message(queue_id).await { - let prev_event = message.next_event().unwrap_or_default(); - - if let Some(item) = &item { - // Cancel delivery for all recipients that match - for rcpt in &mut message.recipients { - if rcpt.address_lcase.contains(item) { - rcpt.status = Status::PermanentFailure(HostResponse { - hostname: ErrorDetails::default(), - response: smtp_proto::Response { - code: 0, - esc: [0, 0, 0], - message: "Delivery canceled.".to_string(), - }, - }); - found = true; - } - } - if found { - // Mark as completed domains without any pending deliveries - for (domain_idx, domain) in - message.domains.iter_mut().enumerate() - { - if matches!( - domain.status, - Status::TemporaryFailure(_) | Status::Scheduled - ) { - let mut total_rcpt = 0; - let mut total_completed = 0; - - for rcpt in &message.recipients { - if rcpt.domain_idx == domain_idx { - total_rcpt += 1; - if matches!( - rcpt.status, - Status::PermanentFailure(_) - | Status::Completed(_) - ) { - total_completed += 1; - } - } - } - - if total_rcpt == total_completed { - domain.status = Status::Completed(()); - } - } - } - - // Delete message if there are no pending deliveries - if message.domains.iter().any(|domain| { - matches!( - domain.status, - Status::TemporaryFailure(_) | Status::Scheduled - ) - }) { - let next_event = - message.next_event().unwrap_or_default(); - message - .save_changes( - self, - next_event.into(), - prev_event.into(), - ) - .await; + if matches { + if offset == 0 { + if limit == 0 || total_returned < limit { + if values { + result_values.push(Message::from(&message)); } else { - message.remove(self, prev_event).await; + result_ids.push(key.deserialize_be_u64(1)?); } + total_returned += 1; } } else { - message.remove(self, prev_event).await; - found = true; + offset -= 1; } + + total += 1; } - result.push(found); - } + Ok(true) + }, + ) + .await; - ( - StatusCode::OK, - serde_json::to_string(&Response { data: result }).unwrap_or_default(), - ) + ( + StatusCode::OK, + if values { + serde_json::to_string(&json!({ + "data": { + "items": result_values, + "total": total, + }, + })) + } else { + serde_json::to_string(&json!({ + "data": { + "items": result_ids, + "total": total, + }, + })) } - Some(error) => error.into_bad_request(), + .unwrap_or_default(), + ) + } + (&Method::GET, "queue", "messages", Some(queue_id)) => { + if let Some(message) = self + .read_message(queue_id.parse().unwrap_or_default()) + .await + { + ( + StatusCode::OK, + serde_json::to_string(&Response { + data: Message::from(&message), + }) + .unwrap_or_default(), + ) + } else { + not_found() } } - (&Method::GET, "report", "list") => { - let mut domain = None; - let mut type_ = None; - let mut error = None; + (&Method::PATCH, "queue", "messages", Some(queue_id)) => { + let time = params + .parse::("at") + .map(|t| t.into_inner()) + .unwrap_or_else(now); + let item = params.get("filter"); - if let Some(query) = uri.query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "type" => match value.as_ref() { - "dmarc" => { - type_ = 0u8.into(); - } - "tls" => { - type_ = 1u8.into(); - } - _ => { - error = format!("Invalid report type {value:?}.").into(); - break; - } - }, - "domain" => { - domain = value.into_owned().into(); - } - _ => { - error = format!("Invalid parameter {key:?}.").into(); - break; + if let Some(mut message) = self + .read_message(queue_id.parse().unwrap_or_default()) + .await + { + let prev_event = message.next_event().unwrap_or_default(); + let mut found = false; + + for domain in &mut message.domains { + if matches!( + domain.status, + Status::Scheduled | Status::TemporaryFailure(_) + ) && item + .as_ref() + .map_or(true, |item| domain.domain.contains(item)) + { + domain.retry.due = time; + if domain.expires > time { + domain.expires = time + 10; } + found = true; } } - } - match error { - None => { - let mut result = Vec::new(); - let from_key = ValueKey::from(ValueClass::Queue( - QueueClass::DmarcReportHeader(ReportEvent { - due: 0, - policy_hash: 0, - seq_id: 0, - domain: String::new(), - }), - )); - let to_key = ValueKey::from(ValueClass::Queue( - QueueClass::TlsReportHeader(ReportEvent { - due: u64::MAX, - policy_hash: 0, - seq_id: 0, - domain: String::new(), - }), - )); - let _ = self - .shared - .default_data_store - .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - if type_.map_or(true, |t| t == *key.last().unwrap()) { - let event = ReportEvent::deserialize(key)?; - if event.seq_id != 0 - && domain.as_ref().map_or(true, |d| { - d.eq_ignore_ascii_case(&event.domain) - }) - { + if found { + let next_event = message.next_event().unwrap_or_default(); + message + .save_changes(self, prev_event.into(), next_event.into()) + .await; + let _ = self.queue.tx.send(queue::Event::Reload).await; + } + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: found }).unwrap_or_default(), + ) + } else { + not_found() + } + } + (&Method::DELETE, "queue", "messages", Some(queue_id)) => { + if let Some(mut message) = self + .read_message(queue_id.parse().unwrap_or_default()) + .await + { + let mut found = false; + let prev_event = message.next_event().unwrap_or_default(); + + if let Some(item) = params.get("filter") { + // Cancel delivery for all recipients that match + for rcpt in &mut message.recipients { + if rcpt.address_lcase.contains(item) { + rcpt.status = Status::PermanentFailure(HostResponse { + hostname: ErrorDetails::default(), + response: smtp_proto::Response { + code: 0, + esc: [0, 0, 0], + message: "Delivery canceled.".to_string(), + }, + }); + found = true; + } + } + if found { + // Mark as completed domains without any pending deliveries + for (domain_idx, domain) in message.domains.iter_mut().enumerate() { + if matches!( + domain.status, + Status::TemporaryFailure(_) | Status::Scheduled + ) { + let mut total_rcpt = 0; + let mut total_completed = 0; + + for rcpt in &message.recipients { + if rcpt.domain_idx == domain_idx { + total_rcpt += 1; + if matches!( + rcpt.status, + Status::PermanentFailure(_) | Status::Completed(_) + ) { + total_completed += 1; + } + } + } + + if total_rcpt == total_completed { + domain.status = Status::Completed(()); + } + } + } + + // Delete message if there are no pending deliveries + if message.domains.iter().any(|domain| { + matches!( + domain.status, + Status::TemporaryFailure(_) | Status::Scheduled + ) + }) { + let next_event = message.next_event().unwrap_or_default(); + message + .save_changes(self, next_event.into(), prev_event.into()) + .await; + } else { + message.remove(self, prev_event).await; + } + } + } else { + message.remove(self, prev_event).await; + found = true; + } + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: found }).unwrap_or_default(), + ) + } else { + not_found() + } + } + (&Method::GET, "queue", "reports", None) => { + let domain = params.get("domain").map(|d| d.to_lowercase()); + let type_ = params.get("type").and_then(|t| match t { + "dmarc" => 0u8.into(), + "tls" => 1u8.into(), + _ => None, + }); + let page: usize = params.parse("page").unwrap_or_default(); + let limit: usize = params.parse("limit").unwrap_or_default(); + + let mut result = Vec::new(); + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportHeader( + ReportEvent { + due: 0, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportHeader( + ReportEvent { + due: u64::MAX, + policy_hash: 0, + seq_id: 0, + domain: String::new(), + }, + ))); + let mut offset = page.saturating_sub(1) * limit; + let mut total = 0; + let mut total_returned = 0; + let _ = self + .shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + if type_.map_or(true, |t| t == *key.last().unwrap()) { + let event = ReportEvent::deserialize(key)?; + if event.seq_id != 0 + && domain.as_ref().map_or(true, |d| event.domain.contains(d)) + { + if offset == 0 { + if limit == 0 || total_returned < limit { result.push( if *key.last().unwrap() == 0 { QueueClass::DmarcReportHeader(event) @@ -814,164 +638,274 @@ impl SMTP { } .queue_id(), ); + total_returned += 1; } + } else { + offset -= 1; } - Ok(true) - }, - ) - .await; - - ( - StatusCode::OK, - serde_json::to_string(&Response { data: result }).unwrap_or_default(), - ) - } - Some(error) => error.into_bad_request(), - } - } - (&Method::GET, "report", "status") => { - let mut report_ids = Vec::new(); - let mut error = None; - - if let Some(query) = uri.query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "id" | "ids" => match value.parse_report_ids() { - Ok(ids) => { - report_ids = ids; + total += 1; } - Err(reason) => { - error = reason.into(); - break; - } - }, - _ => { - error = format!("Invalid parameter {key:?}.").into(); - break; } - } - } - } - let mut result = Vec::with_capacity(report_ids.len()); - for report_id in report_ids { + Ok(true) + }, + ) + .await; + + ( + StatusCode::OK, + serde_json::to_string(&json!({ + "data": { + "items": result, + "total": total, + }, + })) + .unwrap_or_default(), + ) + } + (&Method::GET, "queue", "reports", Some(report_id)) => { + let mut result = None; + if let Some(report_id) = parse_queued_report_id(report_id) { match report_id { QueueClass::DmarcReportHeader(event) => { + let mut rua = Vec::new(); if let Ok(Some(report)) = self - .shared - .default_data_store - .get_value::>(ValueKey::from( - ValueClass::Queue(QueueClass::DmarcReportHeader(event.clone())), - )) + .generate_dmarc_aggregate_report(&event, &mut rua, None) .await { - let mut report = report.inner; - if let Ok(records) = - self.fetch_dmarc_records(&event, &report, None).await - { - report.records = records; - result.push(Report::dmarc(event, report).into()); - continue; - } + result = Report::dmarc(event, report, rua).into(); } } QueueClass::TlsReportHeader(event) => { + let mut rua = Vec::new(); if let Ok(Some(report)) = self - .shared - .default_data_store - .get_value::>(ValueKey::from(ValueClass::Queue( - QueueClass::TlsReportHeader(event.clone()), - ))) + .generate_tls_aggregate_report(&[event.clone()], &mut rua, None) .await { - let mut report = report.inner; - if let Ok(policy) = - self.fetch_tls_policy(&event, report.policy, None).await - { - report.policy = policy.policy; - for record in policy.failure_details { - report.records.push(record.into()); - } - for _ in 0..policy.summary.total_success { - report.records.push(None); - } - result.push(Report::tls(event, report).into()); - continue; - } + result = Report::tls(event, report, rua).into(); } } _ => (), } - - result.push(None); } - match error { - None => ( + if let Some(result) = result { + ( StatusCode::OK, serde_json::to_string(&Response { data: result }).unwrap_or_default(), + ) + } else { + not_found() + } + } + (&Method::DELETE, "queue", "reports", Some(report_id)) => { + if let Some(report_id) = parse_queued_report_id(report_id) { + match report_id { + QueueClass::DmarcReportHeader(event) => { + self.delete_dmarc_report(event).await; + } + QueueClass::TlsReportHeader(event) => { + self.delete_tls_report(vec![event]).await; + } + _ => (), + } + + ( + StatusCode::OK, + serde_json::to_string(&Response { data: true }).unwrap_or_default(), + ) + } else { + not_found() + } + } + (&Method::GET, "reports", class @ ("dmarc" | "tls" | "arf"), None) => { + let filter = params.get("text"); + let page: usize = params.parse::("page").unwrap_or_default(); + let limit: usize = params.parse::("limit").unwrap_or_default(); + + let (from_key, to_key, typ) = match class { + "dmarc" => ( + ValueKey::from(ValueClass::Report(ReportClass::Dmarc { + id: 0, + expires: 0, + })), + ValueKey::from(ValueClass::Report(ReportClass::Dmarc { + id: u64::MAX, + expires: u64::MAX, + })), + ReportType::Dmarc, ), - Some(error) => error.into_bad_request(), - } - } - (&Method::GET, "report", "cancel") => { - let mut report_ids = Vec::new(); - let mut error = None; + "tls" => ( + ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), + ValueKey::from(ValueClass::Report(ReportClass::Tls { + id: u64::MAX, + expires: u64::MAX, + })), + ReportType::Tls, + ), + "arf" => ( + ValueKey::from(ValueClass::Report(ReportClass::Arf { id: 0, expires: 0 })), + ValueKey::from(ValueClass::Report(ReportClass::Arf { + id: u64::MAX, + expires: u64::MAX, + })), + ReportType::Arf, + ), + _ => unreachable!(), + }; - if let Some(query) = uri.query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "id" | "ids" => match value.parse_report_ids() { - Ok(ids) => { - report_ids = ids; + let mut results = Vec::new(); + let mut offset = page.saturating_sub(1) * limit; + let mut total = 0; + let mut last_id = 0; + let result = self + .shared + .default_data_store + .iterate( + IterateParams::new(from_key, to_key) + .set_values(filter.is_some()) + .descending(), + |key, value| { + // Skip chunked records + let id = key.deserialize_be_u64(U64_LEN + 1)?; + if id == last_id { + return Ok(true); + } + last_id = id; + + // TODO: Support filtering chunked records (over 10MB) on FDB + let matches = filter.map_or(true, |filter| match typ { + ReportType::Dmarc => Bincode::< + IncomingReport, + >::deserialize( + value + ) + .map_or(false, |v| v.inner.contains(filter)), + ReportType::Tls => { + Bincode::>::deserialize(value) + .map_or(false, |v| v.inner.contains(filter)) } - Err(reason) => { - error = reason.into(); - break; + ReportType::Arf => { + Bincode::>::deserialize(value) + .map_or(false, |v| v.inner.contains(filter)) } + }); + if matches { + if offset == 0 { + if limit == 0 || results.len() < limit { + results.push(format!( + "{}_{}", + id, + key.deserialize_be_u64(1)? + )); + } + } else { + offset -= 1; + } + + total += 1; + } + + Ok(true) + }, + ) + .await; + match result { + Ok(_) => ( + StatusCode::OK, + serde_json::to_string(&json!({ + "data": { + "items": results, + "total": total, }, - _ => { - error = format!("Invalid parameter {key:?}.").into(); - break; - } - } - } - } - - match error { - None => { - let mut result = Vec::with_capacity(report_ids.len()); - - for report_id in report_ids { - match report_id { - QueueClass::DmarcReportHeader(event) => { - self.delete_dmarc_report(event).await; - } - QueueClass::TlsReportHeader(event) => { - self.delete_tls_report(vec![event]).await; - } - _ => (), - } - - result.push(true); - } - - ( - StatusCode::OK, - serde_json::to_string(&Response { data: result }).unwrap_or_default(), - ) - } - Some(error) => error.into_bad_request(), + })) + .unwrap_or_default(), + ), + Err(err) => err.into_bad_request(), } } - _ => ( - StatusCode::NOT_FOUND, - format!( - "{{\"error\": \"not-found\", \"details\": \"URL {} does not exist.\"}}", - uri.path() - ), - ), + (&Method::GET, "reports", class @ ("dmarc" | "tls" | "arf"), Some(report_id)) => { + if let Some(report_id) = parse_incoming_report_id(class, report_id) { + match &report_id { + ReportClass::Tls { .. } => match self + .shared + .default_data_store + .get_value::>>(ValueKey::from( + ValueClass::Report(report_id), + )) + .await + { + Ok(Some(report)) => ( + StatusCode::OK, + serde_json::to_string(&json!({ + "data": report.inner, + })) + .unwrap_or_default(), + ), + Ok(None) => not_found(), + Err(err) => err.into_bad_request(), + }, + ReportClass::Dmarc { .. } => match self + .shared + .default_data_store + .get_value::>>( + ValueKey::from(ValueClass::Report(report_id)), + ) + .await + { + Ok(Some(report)) => ( + StatusCode::OK, + serde_json::to_string(&json!({ + "data": report.inner, + })) + .unwrap_or_default(), + ), + Ok(None) => not_found(), + Err(err) => err.into_bad_request(), + }, + ReportClass::Arf { .. } => match self + .shared + .default_data_store + .get_value::>>(ValueKey::from( + ValueClass::Report(report_id), + )) + .await + { + Ok(Some(report)) => ( + StatusCode::OK, + serde_json::to_string(&json!({ + "data": report.inner, + })) + .unwrap_or_default(), + ), + Ok(None) => not_found(), + Err(err) => err.into_bad_request(), + }, + } + } else { + not_found() + } + } + (&Method::DELETE, "reports", class @ ("dmarc" | "tls" | "arf"), Some(report_id)) => { + if let Some(report_id) = parse_incoming_report_id(class, report_id) { + let mut batch = BatchBuilder::new(); + batch.clear(ValueClass::Report(report_id)); + let result = self + .shared + .default_data_store + .write(batch.build()) + .await + .is_ok(); + ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), + ) + } else { + not_found() + } + } + _ => not_found(), }; hyper::Response::builder() @@ -986,6 +920,203 @@ impl SMTP { } } +fn not_found() -> (StatusCode, String) { + ( + StatusCode::NOT_FOUND, + "{\"error\": \"not-found\", \"details\": \"URL does not exist.\"}".to_string(), + ) +} + +#[derive(Default)] +struct UrlParams<'x> { + params: HashMap, Cow<'x, str>>, +} + +impl<'x> UrlParams<'x> { + pub fn new(uri: &'x Uri) -> Self { + if let Some(query) = uri.query() { + Self { + params: form_urlencoded::parse(query.as_bytes()) + .filter(|(_, value)| !value.is_empty()) + .collect(), + } + } else { + Self::default() + } + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.params.get(key).map(|v| v.as_ref()) + } + + pub fn has_key(&self, key: &str) -> bool { + self.params.contains_key(key) + } + + pub fn parse(&self, key: &str) -> Option + where + T: std::str::FromStr, + { + self.get(key).and_then(|v| v.parse().ok()) + } +} + +enum ReportType { + Dmarc, + Tls, + Arf, +} + +impl From<&str> for ReportType { + fn from(s: &str) -> Self { + match s { + "dmarc" => Self::Dmarc, + "tls" => Self::Tls, + "arf" => Self::Arf, + _ => unreachable!(), + } + } +} + +trait Contains { + fn contains(&self, text: &str) -> bool; +} + +impl Contains for mail_auth::report::Report { + fn contains(&self, text: &str) -> bool { + self.domain().contains(text) + || self.org_name().to_lowercase().contains(text) + || self.report_id().contains(text) + || self + .extra_contact_info() + .map_or(false, |c| c.to_lowercase().contains(text)) + || self.records().iter().any(|record| record.contains(text)) + } +} + +impl Contains for mail_auth::report::Record { + fn contains(&self, filter: &str) -> bool { + self.envelope_from().contains(filter) + || self.header_from().contains(filter) + || self.envelope_to().map_or(false, |to| to.contains(filter)) + || self.dkim_auth_result().iter().any(|dkim| { + dkim.domain().contains(filter) + || dkim.selector().contains(filter) + || dkim + .human_result() + .as_ref() + .map_or(false, |r| r.contains(filter)) + }) + || self.spf_auth_result().iter().any(|spf| { + spf.domain().contains(filter) + || spf.human_result().map_or(false, |r| r.contains(filter)) + }) + || self + .source_ip() + .map_or(false, |ip| ip.to_string().contains(filter)) + } +} + +impl Contains for TlsReport { + fn contains(&self, text: &str) -> bool { + self.organization_name + .as_ref() + .map_or(false, |o| o.to_lowercase().contains(text)) + || self + .contact_info + .as_ref() + .map_or(false, |c| c.to_lowercase().contains(text)) + || self.report_id.contains(text) + || self.policies.iter().any(|p| p.contains(text)) + } +} + +impl Contains for Policy { + fn contains(&self, filter: &str) -> bool { + self.policy.policy_domain.contains(filter) + || self + .policy + .policy_string + .iter() + .any(|s| s.to_lowercase().contains(filter)) + || self + .policy + .mx_host + .iter() + .any(|s| s.to_lowercase().contains(filter)) + || self.failure_details.iter().any(|f| f.contains(filter)) + } +} + +impl Contains for FailureDetails { + fn contains(&self, filter: &str) -> bool { + self.sending_mta_ip + .map_or(false, |s| s.to_string().contains(filter)) + || self + .receiving_ip + .map_or(false, |s| s.to_string().contains(filter)) + || self + .receiving_mx_hostname + .as_ref() + .map_or(false, |s| s.contains(filter)) + || self + .receiving_mx_helo + .as_ref() + .map_or(false, |s| s.contains(filter)) + || self + .additional_information + .as_ref() + .map_or(false, |s| s.contains(filter)) + || self + .failure_reason_code + .as_ref() + .map_or(false, |s| s.contains(filter)) + } +} + +impl<'x> Contains for Feedback<'x> { + fn contains(&self, text: &str) -> bool { + // Check if any of the string fields contain the filter + self.authentication_results() + .iter() + .any(|s| s.contains(text)) + || self + .original_envelope_id() + .map_or(false, |s| s.contains(text)) + || self + .original_mail_from() + .map_or(false, |s| s.contains(text)) + || self.original_rcpt_to().map_or(false, |s| s.contains(text)) + || self.reported_domain().iter().any(|s| s.contains(text)) + || self.reported_uri().iter().any(|s| s.contains(text)) + || self.reporting_mta().map_or(false, |s| s.contains(text)) + || self.user_agent().map_or(false, |s| s.contains(text)) + || self.dkim_adsp_dns().map_or(false, |s| s.contains(text)) + || self + .dkim_canonicalized_body() + .map_or(false, |s| s.contains(text)) + || self + .dkim_canonicalized_header() + .map_or(false, |s| s.contains(text)) + || self.dkim_domain().map_or(false, |s| s.contains(text)) + || self.dkim_identity().map_or(false, |s| s.contains(text)) + || self.dkim_selector().map_or(false, |s| s.contains(text)) + || self.dkim_selector_dns().map_or(false, |s| s.contains(text)) + || self.spf_dns().map_or(false, |s| s.contains(text)) + || self.message().map_or(false, |s| s.contains(text)) + || self.headers().map_or(false, |s| s.contains(text)) + } +} + +impl Contains for IncomingReport { + fn contains(&self, text: &str) -> bool { + self.from.to_lowercase().contains(text) + || self.to.iter().any(|to| to.to_lowercase().contains(text)) + || self.subject.to_lowercase().contains(text) + || self.report.contains(text) + } +} + impl From<&queue::Message> for Message { fn from(message: &queue::Message) -> Self { let now = now(); @@ -1049,23 +1180,25 @@ impl From<&queue::Message> for Message { } impl Report { - fn dmarc(event: ReportEvent, report: DmarcFormat) -> Self { + fn dmarc(event: ReportEvent, report: report::Report, rua: Vec) -> Self { Self::Dmarc { domain: event.domain.clone(), range_from: DateTime::from_timestamp(event.seq_id as i64), range_to: DateTime::from_timestamp(event.due as i64), id: QueueClass::DmarcReportHeader(event).queue_id(), report, + rua, } } - fn tls(event: ReportEvent, report: TlsFormat) -> Self { + fn tls(event: ReportEvent, report: TlsReport, rua: Vec) -> Self { Self::Tls { domain: event.domain.clone(), range_from: DateTime::from_timestamp(event.seq_id as i64), range_to: DateTime::from_timestamp(event.due as i64), id: QueueClass::TlsReportHeader(event).queue_id(), report, + rua, } } } @@ -1088,80 +1221,54 @@ impl GenerateQueueId for QueueClass { } } -trait ParseValues { - fn parse_timestamp(&self) -> Result; - fn parse_queue_ids(&self) -> Result, String>; - fn parse_report_ids(&self) -> Result, String>; +fn parse_queued_report_id(id: &str) -> Option { + let mut parts = id.split('!'); + let type_ = parts.next()?; + let event = ReportEvent { + domain: parts.next()?.to_string(), + policy_hash: parts.next().and_then(|p| p.parse::().ok())?, + seq_id: parts.next().and_then(|p| p.parse::().ok())?, + due: parts.next().and_then(|p| p.parse::().ok())?, + }; + match type_ { + "d" => Some(QueueClass::DmarcReportHeader(event)), + "t" => Some(QueueClass::TlsReportHeader(event)), + _ => None, + } } -impl ParseValues for Cow<'_, str> { - fn parse_timestamp(&self) -> Result { - if let Some(dt) = DateTime::parse_rfc3339(self.as_ref()) { +fn parse_incoming_report_id(class: &str, id: &str) -> Option { + let mut parts = id.split('_'); + let id = parts.next()?.parse().ok()?; + let expires = parts.next()?.parse().ok()?; + match class { + "dmarc" => Some(ReportClass::Dmarc { id, expires }), + "tls" => Some(ReportClass::Tls { id, expires }), + "arf" => Some(ReportClass::Arf { id, expires }), + _ => None, + } +} + +struct Timestamp(u64); + +impl FromStr for Timestamp { + type Err = (); + + fn from_str(s: &str) -> Result { + if let Some(dt) = DateTime::parse_rfc3339(s) { let instant = dt.to_timestamp() as u64; if instant >= now() { - return Ok(instant); + return Ok(Timestamp(instant)); } } - Err(format!("Invalid timestamp {self:?}.")) + Err(()) } +} - fn parse_queue_ids(&self) -> Result, String> { - let mut ids = Vec::new(); - for id in self.split(',') { - if !id.is_empty() { - match id.parse() { - Ok(id) => { - ids.push(id); - } - Err(_) => { - return Err(format!("Failed to parse id {id:?}.")); - } - } - } - } - Ok(ids) - } - - fn parse_report_ids(&self) -> Result, String> { - let mut ids = Vec::new(); - for id in self.split(',') { - if !id.is_empty() { - let mut parts = id.split('!'); - match ( - parts.next(), - parts.next(), - parts.next().and_then(|p| p.parse::().ok()), - parts.next().and_then(|p| p.parse::().ok()), - parts.next().and_then(|p| p.parse::().ok()), - ) { - (Some("d"), Some(domain), Some(policy), Some(seq_id), Some(due)) - if !domain.is_empty() => - { - ids.push(QueueClass::DmarcReportHeader(ReportEvent { - due, - policy_hash: policy, - seq_id, - domain: domain.to_string(), - })); - } - (Some("t"), Some(domain), Some(policy), Some(seq_id), Some(due)) - if !domain.is_empty() => - { - ids.push(QueueClass::TlsReportHeader(ReportEvent { - due, - policy_hash: policy, - seq_id, - domain: domain.to_string(), - })); - } - _ => { - return Err(format!("Failed to parse id {id:?}.")); - } - } - } - } - Ok(ids) +impl Timestamp { + pub fn into_inner(self) -> u64 { + self.0 } } @@ -1181,6 +1288,19 @@ impl BadRequest for String { } } +impl BadRequest for store::Error { + fn into_bad_request(self) -> (StatusCode, String) { + ( + StatusCode::INTERNAL_SERVER_ERROR, + serde_json::to_string(&json!({ + "error": "internal-error", + "details": self.to_string(), + })) + .unwrap_or_default(), + ) + } +} + fn is_zero(num: &i16) -> bool { *num == 0 } diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index f1adb122..252c187b 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -25,7 +25,7 @@ use std::{ borrow::Cow, collections::hash_map::Entry, io::{Cursor, Read}, - sync::{atomic::Ordering, Arc}, + sync::Arc, time::SystemTime, }; @@ -37,6 +37,12 @@ use mail_auth::{ }; use mail_parser::{DateTime, MessageParser, MimeHeaders, PartType}; +use store::{ + write::{now, BatchBuilder, Bincode, ReportClass, ValueClass}, + Serialize, +}; +use tokio::runtime::Handle; + use crate::core::SMTP; enum Compression { @@ -45,18 +51,26 @@ enum Compression { Zip, } -enum Format { - Dmarc, - Tls, - Arf, +enum Format { + Dmarc(D), + Tls(T), + Arf(A), } struct ReportData<'x> { compression: Compression, - format: Format, + format: Format<(), (), ()>, data: &'x [u8], } +#[derive(serde::Serialize, serde::Deserialize)] +pub struct IncomingReport { + pub from: String, + pub to: Vec, + pub subject: String, + pub report: T, +} + pub trait AnalyzeReport { fn analyze_report(&self, message: Arc>); } @@ -64,6 +78,7 @@ pub trait AnalyzeReport { impl AnalyzeReport for Arc { fn analyze_report(&self, message: Arc>) { let core = self.clone(); + let handle = Handle::current(); self.worker_pool.spawn(move || { let message = if let Some(message) = MessageParser::default().parse(message.as_ref()) { message @@ -75,7 +90,15 @@ impl AnalyzeReport for Arc { .from() .and_then(|a| a.last()) .and_then(|a| a.address()) - .unwrap_or("unknown"); + .unwrap_or_default() + .to_string(); + let to = message.to().map_or_else(Vec::new, |a| { + a.iter() + .filter_map(|a| a.address()) + .map(|a| a.to_string()) + .collect() + }); + let subject = message.subject().unwrap_or_default().to_string(); let mut reports = Vec::new(); for part in &message.parts { @@ -92,13 +115,13 @@ impl AnalyzeReport for Arc { { reports.push(ReportData { compression: Compression::None, - format: Format::Dmarc, + format: Format::Dmarc(()), data: report.as_bytes(), }); } else if part.is_content_type("message", "feedback-report") { reports.push(ReportData { compression: Compression::None, - format: Format::Arf, + format: Format::Arf(()), data: report.as_bytes(), }); } @@ -107,7 +130,7 @@ impl AnalyzeReport for Arc { if part.is_content_type("message", "feedback-report") { reports.push(ReportData { compression: Compression::None, - format: Format::Arf, + format: Format::Arf(()), data: report.as_ref(), }); continue; @@ -131,13 +154,13 @@ impl AnalyzeReport for Arc { _ => Compression::None, }; let format = match (tls_parts.map(|(c, _)| c).unwrap_or(subtype), ext) { - ("xml", _) => Format::Dmarc, - ("tlsrpt", _) | (_, "json") => Format::Tls, + ("xml", _) => Format::Dmarc(()), + ("tlsrpt", _) | (_, "json") => Format::Tls(()), _ => { if attachment_name .map_or(false, |n| n.contains(".xml") || n.contains('!')) { - Format::Dmarc + Format::Dmarc(()) } else { continue; } @@ -213,10 +236,11 @@ impl AnalyzeReport for Arc { } }; - match report.format { - Format::Dmarc => match Report::parse_xml(&data) { + let report = match report.format { + Format::Dmarc(_) => match Report::parse_xml(&data) { Ok(report) => { report.log(); + Format::Dmarc(report) } Err(err) => { tracing::debug!( @@ -228,9 +252,10 @@ impl AnalyzeReport for Arc { continue; } }, - Format::Tls => match TlsReport::parse_json(&data) { + Format::Tls(_) => match TlsReport::parse_json(&data) { Ok(report) => { report.log(); + Format::Tls(report) } Err(err) => { tracing::debug!( @@ -242,9 +267,10 @@ impl AnalyzeReport for Arc { continue; } }, - Format::Arf => match Feedback::parse_arf(&data) { + Format::Arf(_) => match Feedback::parse_arf(&data) { Some(report) => { report.log(); + Format::Arf(report.into_owned()) } None => { tracing::debug!( @@ -255,47 +281,72 @@ impl AnalyzeReport for Arc { continue; } }, - } + }; - // Save report - if let Some(report_path) = &core.report.config.analysis.store { - let (report_format, extension) = match report.format { - Format::Dmarc => ("dmarc", "xml"), - Format::Tls => ("tlsrpt", "json"), - Format::Arf => ("arf", "txt"), - }; - let c_extension = match report.compression { - Compression::None => "", - Compression::Gzip => ".gz", - Compression::Zip => ".zip", - }; - let now = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .map_or(0, |d| d.as_secs()); + // Store report + if let Some(expires_in) = &core.report.config.analysis.store { + let expires = now() + expires_in.as_secs(); let id = core .report .config .analysis .report_id - .fetch_add(1, Ordering::Relaxed); + .generate() + .unwrap_or(expires); - // Build path - let mut report_path = report_path.clone(); - report_path.push(format!( - "{report_format}_{now}_{id}.{extension}{c_extension}" - )); - if let Err(err) = std::fs::write(&report_path, report.data) { - tracing::warn!( - context = "report", - event = "error", - from = from, - "Failed to write incoming report to {}: {}", - report_path.display(), - err - ); + let mut batch = BatchBuilder::new(); + match report { + Format::Dmarc(report) => { + batch.set( + ValueClass::Report(ReportClass::Dmarc { id, expires }), + Bincode::new(IncomingReport { + from, + to, + subject, + report, + }) + .serialize(), + ); + } + Format::Tls(report) => { + batch.set( + ValueClass::Report(ReportClass::Tls { id, expires }), + Bincode::new(IncomingReport { + from, + to, + subject, + report, + }) + .serialize(), + ); + } + Format::Arf(report) => { + batch.set( + ValueClass::Report(ReportClass::Arf { id, expires }), + Bincode::new(IncomingReport { + from, + to, + subject, + report, + }) + .serialize(), + ); + } } + let batch = batch.build(); + let _enter = handle.enter(); + handle.spawn(async move { + if let Err(err) = core.shared.default_data_store.write(batch).await { + tracing::warn!( + context = "report", + event = "error", + "Failed to write incoming report: {}", + err + ); + } + }); } - break; + return; } }); } diff --git a/crates/smtp/src/reporting/dkim.rs b/crates/smtp/src/reporting/dkim.rs index 23a78170..3f6fb0a0 100644 --- a/crates/smtp/src/reporting/dkim.rs +++ b/crates/smtp/src/reporting/dkim.rs @@ -73,7 +73,7 @@ impl Session { .with_dkim_domain(signature.domain()) .with_dkim_selector(signature.selector()) .with_dkim_identity(signature.identity()) - .with_headers(message.raw_headers()) + .with_headers(std::str::from_utf8(message.raw_headers()).unwrap_or_default()) .write_rfc5322( ( self.core diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index 65dd7d3b..a10f2f8d 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -129,7 +129,7 @@ impl Session { let mut auth_failure = self .new_auth_failure(AuthFailureType::Dmarc, rejected) .with_authentication_results(auth_results.to_string()) - .with_headers(message.raw_headers()); + .with_headers(std::str::from_utf8(message.raw_headers()).unwrap_or_default()); // Report the first failed signature let dkim_failed = if let ( @@ -300,7 +300,7 @@ impl Session { } impl SMTP { - pub async fn generate_dmarc_report(&self, event: ReportEvent) { + pub async fn send_dmarc_aggregate_report(&self, event: ReportEvent) { let span = tracing::info_span!( "dmarc-report", domain = event.domain, @@ -308,16 +308,21 @@ impl SMTP { range_to = event.due, ); - // Deserialize report - let dmarc = match self - .shared - .default_data_store - .get_value::>(ValueKey::from(ValueClass::Queue( - QueueClass::DmarcReportHeader(event.clone()), - ))) + // Generate report + let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( + self.eval_if( + &self.report.config.dmarc_aggregate.max_size, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or(25 * 1024 * 1024), + )); + let mut rua = Vec::new(); + let report = match self + .generate_dmarc_aggregate_report(&event, &mut rua, Some(&mut serialized_size)) .await { - Ok(Some(dmarc)) => dmarc.inner, + Ok(Some(report)) => report, Ok(None) => { tracing::warn!( parent: &span, @@ -330,7 +335,7 @@ impl SMTP { tracing::warn!( parent: &span, event = "error", - "Failed to read DMARC report: {}", + "Failed to read DMARC records: {}", err ); return; @@ -341,7 +346,7 @@ impl SMTP { let rua = match self .resolvers .dns - .verify_dmarc_report_address(&event.domain, &dmarc.rua) + .verify_dmarc_report_address(&event.domain, &rua) .await { Some(rcpts) => { @@ -355,7 +360,7 @@ impl SMTP { parent: &span, event = "failed", reason = "unauthorized-rua", - rua = ?dmarc.rua, + rua = ?rua, "Unauthorized external reporting addresses" ); self.delete_dmarc_report(event).await; @@ -367,7 +372,7 @@ impl SMTP { parent: &span, event = "failed", reason = "dns-failure", - rua = ?dmarc.rua, + rua = ?rua, "Failed to validate external report addresses", ); self.delete_dmarc_report(event).await; @@ -375,69 +380,8 @@ impl SMTP { } }; - let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( - self.eval_if( - &self.report.config.dmarc_aggregate.max_size, - &RecipientDomain::new(event.domain.as_str()), - ) - .await - .unwrap_or(25 * 1024 * 1024), - )); - let _ = serde::Serialize::serialize(&dmarc, &mut serialized_size); + // Serialize report let config = &self.report.config.dmarc_aggregate; - - // Fetch records - let records = match self - .fetch_dmarc_records(&event, &dmarc, Some(&mut serialized_size)) - .await - { - Ok(records) => records, - Err(err) => { - tracing::warn!( - parent: &span, - event = "error", - "Failed to read DMARC records: {}", - err - ); - return; - } - }; - - // Create report - let mut report = Report::new() - .with_policy_published(dmarc.policy) - .with_date_range_begin(event.seq_id) - .with_date_range_end(event.due) - .with_report_id(format!("{}_{}", event.policy_hash, event.seq_id)) - .with_email( - self.eval_if( - &config.address, - &RecipientDomain::new(event.domain.as_str()), - ) - .await - .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), - ); - if let Some(org_name) = self - .eval_if::( - &config.org_name, - &RecipientDomain::new(event.domain.as_str()), - ) - .await - { - report = report.with_org_name(org_name); - } - if let Some(contact_info) = self - .eval_if::( - &config.contact_info, - &RecipientDomain::new(event.domain.as_str()), - ) - .await - { - report = report.with_extra_contact_info(contact_info); - } - for record in records { - report = report.with_record(record); - } let from_addr = self .eval_if( &config.address, @@ -472,12 +416,66 @@ impl SMTP { self.delete_dmarc_report(event).await; } - pub(crate) async fn fetch_dmarc_records( + pub(crate) async fn generate_dmarc_aggregate_report( &self, event: &ReportEvent, - dmarc: &DmarcFormat, + rua: &mut Vec, mut serialized_size: Option<&mut serde_json::Serializer>, - ) -> store::Result> { + ) -> store::Result> { + // Deserialize report + let dmarc = match self + .shared + .default_data_store + .get_value::>(ValueKey::from(ValueClass::Queue( + QueueClass::DmarcReportHeader(event.clone()), + ))) + .await? + { + Some(dmarc) => dmarc.inner, + None => { + return Ok(None); + } + }; + let _ = std::mem::replace(rua, dmarc.rua); + + // Create report + let config = &self.report.config.dmarc_aggregate; + let mut report = Report::new() + .with_policy_published(dmarc.policy) + .with_date_range_begin(event.seq_id) + .with_date_range_end(event.due) + .with_report_id(format!("{}_{}", event.policy_hash, event.seq_id)) + .with_email( + self.eval_if( + &config.address, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()), + ); + if let Some(org_name) = self + .eval_if::( + &config.org_name, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + { + report = report.with_org_name(org_name); + } + if let Some(contact_info) = self + .eval_if::( + &config.contact_info, + &RecipientDomain::new(event.domain.as_str()), + ) + .await + { + report = report.with_extra_contact_info(contact_info); + } + + if let Some(serialized_size) = serialized_size.as_deref_mut() { + let _ = serde::Serialize::serialize(&report, serialized_size); + } + // Group duplicates let from_key = ValueKey::from(ValueClass::Queue(QueueClass::DmarcReportEvent( ReportEvent { @@ -522,12 +520,11 @@ impl SMTP { ) .await?; - let mut records = Vec::with_capacity(record_map.len()); for (record, count) in record_map { - records.push(record.with_count(count)); + report = report.with_record(record.with_count(count)); } - Ok(records) + Ok(Some(report)) } pub async fn delete_dmarc_report(&self, event: ReportEvent) { diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index cc834850..70f38416 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -311,6 +311,7 @@ impl SerializedSize { impl io::Write for SerializedSize { fn write(&mut self, buf: &[u8]) -> io::Result { + //let c = print!(" (left: {}, buf: {})", self.bytes_left, buf.len()); let buf_len = buf.len(); if buf_len <= self.bytes_left { self.bytes_left -= buf_len; diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs index f05592b7..b5bd0098 100644 --- a/crates/smtp/src/reporting/scheduler.rs +++ b/crates/smtp/src/reporting/scheduler.rs @@ -70,7 +70,7 @@ impl SpawnReport for mpsc::Receiver { match report_event { QueueClass::DmarcReportHeader(event) if event.due <= now => { if core_.try_lock_report(QueueClass::dmarc_lock(&event)).await { - core_.generate_dmarc_report(event).await; + core_.send_dmarc_aggregate_report(event).await; } } QueueClass::TlsReportHeader(event) if event.due <= now => { @@ -83,12 +83,12 @@ impl SpawnReport for mpsc::Receiver { } } - for (domain_name, tls_report) in tls_reports { + for (_, tls_report) in tls_reports { if core_ .try_lock_report(QueueClass::tls_lock(tls_report.first().unwrap())) .await { - core_.generate_tls_report(domain_name, tls_report).await; + core_.send_tls_aggregate_report(tls_report).await; } } }); diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 64843de9..827f1087 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -67,10 +67,10 @@ pub struct TlsFormat { pub static TLS_HTTP_REPORT: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); impl SMTP { - pub async fn generate_tls_report(&self, domain_name: String, events: Vec) { - let (event_from, event_to, policy) = events + pub async fn send_tls_aggregate_report(&self, events: Vec) { + let (domain_name, event_from, event_to) = events .first() - .map(|e| (e.seq_id, e.due, e.policy_hash)) + .map(|e| (e.domain.as_str(), e.seq_id, e.due)) .unwrap(); let span = tracing::info_span!( @@ -80,107 +80,41 @@ impl SMTP { range_to = event_to, ); - // Deserialize report - let config = &self.report.config.tls; - let mut report = TlsReport { - organization_name: self - .eval_if( - &config.org_name, - &RecipientDomain::new(domain_name.as_str()), - ) - .await - .clone(), - date_range: DateRange { - start_datetime: DateTime::from_timestamp(event_from as i64), - end_datetime: DateTime::from_timestamp(event_to as i64), - }, - contact_info: self - .eval_if( - &config.contact_info, - &RecipientDomain::new(domain_name.as_str()), - ) - .await - .clone(), - report_id: format!("{}_{}", event_from, policy), - policies: Vec::with_capacity(events.len()), - }; + // Generate report let mut rua = Vec::new(); let mut serialized_size = serde_json::Serializer::new(SerializedSize::new( self.eval_if( &self.report.config.tls.max_size, - &RecipientDomain::new(domain_name.as_str()), + &RecipientDomain::new(domain_name), ) .await .unwrap_or(25 * 1024 * 1024), )); - let _ = serde::Serialize::serialize(&report, &mut serialized_size); - - for event in &events { - // Deserialize report - let tls = match self - .shared - .default_data_store - .get_value::>(ValueKey::from(ValueClass::Queue( - QueueClass::TlsReportHeader(event.clone()), - ))) - .await - { - Ok(Some(dmarc)) => dmarc.inner, - Ok(None) => { - tracing::warn!( - parent: &span, - event = "missing", - "Failed to read DMARC report: Report not found" - ); - continue; - } - Err(err) => { - tracing::warn!( - parent: &span, - event = "error", - "Failed to read DMARC report: {}", - err - ); - continue; - } - }; - let _ = serde::Serialize::serialize(&tls, &mut serialized_size); - - // Fetch policy - match self - .fetch_tls_policy(event, tls.policy, Some(&mut serialized_size)) - .await - { - Ok(policy) => { - report.policies.push(policy); - for entry in tls.rua { - if !rua.contains(&entry) { - rua.push(entry); - } - } - } - Err(err) => { - tracing::warn!( - parent: &span, - event = "error", - "Failed to read TLS report: {}", - err - ); - return; - } + let report = match self + .generate_tls_aggregate_report(&events, &mut rua, Some(&mut serialized_size)) + .await + { + Ok(Some(report)) => report, + Ok(None) => { + // This should not happen + tracing::warn!( + parent: &span, + event = "empty-report", + "No policies found in report" + ); + self.delete_tls_report(events).await; + return; } - } - - if report.policies.is_empty() { - // This should not happen - tracing::warn!( - parent: &span, - event = "empty-report", - "No policies found in report" - ); - self.delete_tls_report(events).await; - return; - } + Err(err) => { + tracing::warn!( + parent: &span, + event = "error", + "Failed to read TLS report: {}", + err + ); + return; + } + }; // Compress and serialize report let json = report.to_json(); @@ -264,22 +198,23 @@ impl SMTP { // Deliver report over SMTP if !rcpts.is_empty() { + let config = &self.report.config.tls; let from_addr = self - .eval_if(&config.address, &RecipientDomain::new(domain_name.as_str())) + .eval_if(&config.address, &RecipientDomain::new(domain_name)) .await .unwrap_or_else(|| "MAILER-DAEMON@localhost".to_string()); let mut message = Vec::with_capacity(2048); let _ = report.write_rfc5322_from_bytes( - &domain_name, + domain_name, &self .eval_if( &self.report.config.submitter, - &RecipientDomain::new(domain_name.as_str()), + &RecipientDomain::new(domain_name), ) .await .unwrap_or_else(|| "localhost".to_string()), ( - self.eval_if(&config.name, &RecipientDomain::new(domain_name.as_str())) + self.eval_if(&config.name, &RecipientDomain::new(domain_name)) .await .unwrap_or_else(|| "Mail Delivery Subsystem".to_string()) .as_str(), @@ -310,76 +245,139 @@ impl SMTP { self.delete_tls_report(events).await; } - pub(crate) async fn fetch_tls_policy( + pub(crate) async fn generate_tls_aggregate_report( &self, - event: &ReportEvent, - policy_details: PolicyDetails, + events: &[ReportEvent], + rua: &mut Vec, mut serialized_size: Option<&mut serde_json::Serializer>, - ) -> store::Result { - // Group duplicates - let mut total_success = 0; - let mut total_failure = 0; + ) -> store::Result> { + let (domain_name, event_from, event_to, policy) = events + .first() + .map(|e| (e.domain.as_str(), e.seq_id, e.due, e.policy_hash)) + .unwrap(); + let config = &self.report.config.tls; + let mut report = TlsReport { + organization_name: self + .eval_if(&config.org_name, &RecipientDomain::new(domain_name)) + .await + .clone(), + date_range: DateRange { + start_datetime: DateTime::from_timestamp(event_from as i64), + end_datetime: DateTime::from_timestamp(event_to as i64), + }, + contact_info: self + .eval_if(&config.contact_info, &RecipientDomain::new(domain_name)) + .await + .clone(), + report_id: format!("{}_{}", event_from, policy), + policies: Vec::with_capacity(events.len()), + }; - let from_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { - due: event.due, - policy_hash: event.policy_hash, - seq_id: 0, - domain: event.domain.clone(), - }))); - let to_key = ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { - due: event.due, - policy_hash: event.policy_hash, - seq_id: u64::MAX, - domain: event.domain.clone(), - }))); - let mut record_map = AHashMap::new(); - self.shared - .default_data_store - .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { - if let Some(failure_details) = - Bincode::>::deserialize(v)?.inner - { - match record_map.entry(failure_details) { - Entry::Occupied(mut e) => { - total_failure += 1; - *e.get_mut() += 1; - Ok(true) - } - Entry::Vacant(e) => { - if serialized_size - .as_deref_mut() - .map_or(true, |serialized_size| { - serde::Serialize::serialize(e.key(), serialized_size).is_ok() - }) - { + if let Some(serialized_size) = serialized_size.as_deref_mut() { + let _ = serde::Serialize::serialize(&report, serialized_size); + } + + for event in events { + let tls = if let Some(tls) = self + .shared + .default_data_store + .get_value::>(ValueKey::from(ValueClass::Queue( + QueueClass::TlsReportHeader(event.clone()), + ))) + .await? + { + tls.inner + } else { + continue; + }; + + if let Some(serialized_size) = serialized_size.as_deref_mut() { + if serde::Serialize::serialize(&tls, serialized_size).is_err() { + continue; + } + } + + // Group duplicates + let mut total_success = 0; + let mut total_failure = 0; + let from_key = + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: 0, + domain: event.domain.clone(), + }))); + let to_key = + ValueKey::from(ValueClass::Queue(QueueClass::TlsReportEvent(ReportEvent { + due: event.due, + policy_hash: event.policy_hash, + seq_id: u64::MAX, + domain: event.domain.clone(), + }))); + let mut record_map = AHashMap::new(); + self.shared + .default_data_store + .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { + if let Some(failure_details) = + Bincode::>::deserialize(v)?.inner + { + match record_map.entry(failure_details) { + Entry::Occupied(mut e) => { total_failure += 1; - e.insert(1u32); + *e.get_mut() += 1; Ok(true) - } else { - Ok(false) + } + Entry::Vacant(e) => { + if serialized_size + .as_deref_mut() + .map_or(true, |serialized_size| { + serde::Serialize::serialize(e.key(), serialized_size) + .is_ok() + }) + { + total_failure += 1; + e.insert(1u32); + Ok(true) + } else { + Ok(false) + } } } + } else { + total_success += 1; + Ok(true) } - } else { - total_success += 1; - Ok(true) - } - }) - .await?; - - Ok(Policy { - policy: policy_details, - summary: Summary { - total_success, - total_failure, - }, - failure_details: record_map - .into_iter() - .map(|(mut r, count)| { - r.failed_session_count = count; - r }) - .collect(), + .await?; + + // Add policy + report.policies.push(Policy { + policy: tls.policy, + summary: Summary { + total_success, + total_failure, + }, + failure_details: record_map + .into_iter() + .map(|(mut r, count)| { + r.failed_session_count = count; + r + }) + .collect(), + }); + + // Add report URIs + for entry in tls.rua { + if !rua.contains(&entry) { + rua.push(entry); + } + } + } + + Ok(if !report.policies.is_empty() { + Some(report) + } else { + None }) } diff --git a/crates/smtp/src/scripts/functions/unicode.rs b/crates/smtp/src/scripts/functions/unicode.rs index c11721e3..2829cddd 100644 --- a/crates/smtp/src/scripts/functions/unicode.rs +++ b/crates/smtp/src/scripts/functions/unicode.rs @@ -87,7 +87,10 @@ impl CharUtils for char { } pub fn fn_cure_text<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { - decancer::cure(v[0].to_string().as_ref()).into_str().into() + decancer::cure(v[0].to_string().as_ref(), decancer::Options::default()) + .map(|s| s.into_str()) + .unwrap_or_default() + .into() } pub fn fn_unicode_skeleton<'x>(_: &'x Context<'x, SieveContext>, v: Vec) -> Variable { diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 2a0a40fc..4905d679 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -9,8 +9,8 @@ utils = { path = "../utils" } nlp = { path = "../nlp" } rocksdb = { version = "0.22", optional = true, features = ["multi-threaded-cf"] } foundationdb = { version = "0.8.0", features = ["embedded-fdb-include"], optional = true } -rusqlite = { version = "0.30.0", features = ["bundled"], optional = true } -rust-s3 = { version = "0.33.0", default-features = false, features = ["tokio-rustls-tls"], optional = true } +rusqlite = { version = "0.31.0", features = ["bundled"], optional = true } +rust-s3 = { version = "0.33.0", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true } tokio = { version = "1.23", features = ["sync", "fs", "io-util"] } r2d2 = { version = "0.8.10", optional = true } futures = { version = "0.3", optional = true } diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index f8e27558..9fe091a2 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -390,7 +390,7 @@ impl FdbStore { } } - pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { + pub(crate) async fn purge_store(&self) -> crate::Result<()> { // Obtain all empty bitmaps let trx = self.db.create_trx()?; let mut iter = trx.get_ranges( diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 419ed37c..bd0c1a1c 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -270,7 +270,7 @@ impl MysqlStore { trx.commit().await.map(|_| true) } - pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { + pub(crate) async fn purge_store(&self) -> crate::Result<()> { let mut conn = self.conn_pool.get_conn().await?; let s = conn diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 9846fcd0..fa86df72 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -287,7 +287,7 @@ impl PostgresStore { trx.commit().await.map(|_| true) } - pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { + pub(crate) async fn purge_store(&self) -> crate::Result<()> { let conn = self.conn_pool.get().await?; let s = conn diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index bf4e393e..0770a650 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -120,7 +120,7 @@ impl RocksDbStore { .await } - pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { + pub(crate) async fn purge_store(&self) -> crate::Result<()> { let db = self.db.clone(); self.spawn_worker(move || { let cf = db diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index 27ea4690..83fb7622 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -203,7 +203,7 @@ impl SqliteStore { .await } - pub(crate) async fn purge_bitmaps(&self) -> crate::Result<()> { + pub(crate) async fn purge_store(&self) -> crate::Result<()> { let conn = self.conn_pool.get()?; self.spawn_worker(move || { conn.prepare_cached(&format!( diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 5b1dd3a6..e0eeb360 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -228,7 +228,7 @@ impl ConfigStore for Config { schedules.push(PurgeSchedule { cron, store_id: store_id.to_string(), - store: PurgeStore::Bitmaps(store.clone()), + store: PurgeStore::Data(store.clone()), }); } diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index b357a3ba..6f23e586 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -294,7 +294,7 @@ impl LookupStore { } } - pub async fn purge_expired(&self) -> crate::Result<()> { + pub async fn purge_lookup_store(&self) -> crate::Result<()> { match self { LookupStore::Store(store) => { // Delete expired keys diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index fe9ed1b5..881a599b 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -26,7 +26,7 @@ use std::ops::{BitAndAssign, Range}; use roaring::RoaringBitmap; use crate::{ - write::{key::KeySerializer, AnyKey, Batch, BitmapClass, ValueClass}, + write::{key::KeySerializer, now, AnyKey, Batch, BitmapClass, ReportClass, ValueClass}, BitmapKey, Deserialize, IterateParams, Key, Store, ValueKey, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, U32_LEN, }; @@ -241,18 +241,45 @@ impl Store { } } - pub async fn purge_bitmaps(&self) -> crate::Result<()> { + pub async fn purge_store(&self) -> crate::Result<()> { + // Delete expired reports + let now = now(); + self.delete_range( + ValueKey::from(ValueClass::Report(ReportClass::Dmarc { id: 0, expires: 0 })), + ValueKey::from(ValueClass::Report(ReportClass::Dmarc { + id: u64::MAX, + expires: now, + })), + ) + .await?; + self.delete_range( + ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), + ValueKey::from(ValueClass::Report(ReportClass::Tls { + id: u64::MAX, + expires: now, + })), + ) + .await?; + self.delete_range( + ValueKey::from(ValueClass::Report(ReportClass::Arf { id: 0, expires: 0 })), + ValueKey::from(ValueClass::Report(ReportClass::Arf { + id: u64::MAX, + expires: now, + })), + ) + .await?; + match self { #[cfg(feature = "sqlite")] - Self::SQLite(store) => store.purge_bitmaps().await, + Self::SQLite(store) => store.purge_store().await, #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.purge_bitmaps().await, + Self::FoundationDb(store) => store.purge_store().await, #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.purge_bitmaps().await, + Self::PostgreSQL(store) => store.purge_store().await, #[cfg(feature = "mysql")] - Self::MySQL(store) => store.purge_bitmaps().await, + Self::MySQL(store) => store.purge_store().await, #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.purge_bitmaps().await, + Self::RocksDb(store) => store.purge_store().await, } } @@ -462,7 +489,7 @@ impl Store { self.blob_expire_all().await; self.purge_blobs(blob_store).await.unwrap(); - self.purge_bitmaps().await.unwrap(); + self.purge_store().await.unwrap(); let store = self.clone(); let mut failed = false; diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 1aa586dc..b1ace3b6 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -31,8 +31,8 @@ use crate::{ }; use super::{ - AnyKey, BitmapClass, BlobOp, DirectoryClass, LookupClass, QueueClass, ReportEvent, TagValue, - ValueClass, + AnyKey, BitmapClass, BlobOp, DirectoryClass, LookupClass, QueueClass, ReportClass, ReportEvent, + TagValue, ValueClass, }; pub struct KeySerializer { @@ -348,6 +348,17 @@ impl + Sync + Send> Key for ValueKey { QueueClass::QuotaCount(key) => serializer.write(55u8).write(key.as_slice()), QueueClass::QuotaSize(key) => serializer.write(56u8).write(key.as_slice()), }, + ValueClass::Report(report) => match report { + ReportClass::Tls { id, expires } => { + serializer.write(60u8).write(*expires).write(*id) + } + ReportClass::Dmarc { id, expires } => { + serializer.write(61u8).write(*expires).write(*id) + } + ReportClass::Arf { id, expires } => { + serializer.write(62u8).write(*expires).write(*id) + } + }, } .finalize() } @@ -503,6 +514,7 @@ impl ValueClass { } QueueClass::QuotaCount(v) | QueueClass::QuotaSize(v) => v.len(), }, + ValueClass::Report(_) => U64_LEN * 2 + 1, } } } diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index a08b514e..7db6e2dd 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -140,6 +140,7 @@ pub enum ValueClass { IndexEmail(u64), Config(Vec), Queue(QueueClass), + Report(ReportClass), } #[derive(Debug, PartialEq, Clone, Eq, Hash)] @@ -172,6 +173,13 @@ pub enum QueueClass { QuotaSize(Vec), } +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub enum ReportClass { + Tls { id: u64, expires: u64 }, + Dmarc { id: u64, expires: u64 }, + Arf { id: u64, expires: u64 }, +} + #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub struct QueueEvent { pub due: u64, diff --git a/crates/store/src/write/purge.rs b/crates/store/src/write/purge.rs index 6d7898f6..3ffd6fb6 100644 --- a/crates/store/src/write/purge.rs +++ b/crates/store/src/write/purge.rs @@ -29,7 +29,7 @@ use utils::config::cron::SimpleCron; use crate::{BlobStore, LookupStore, Store}; pub enum PurgeStore { - Bitmaps(Store), + Data(Store), Blobs { store: Store, blob_store: BlobStore }, Lookup(LookupStore), } @@ -62,11 +62,11 @@ impl PurgeSchedule { } let result = match &self.store { - PurgeStore::Bitmaps(store) => store.purge_bitmaps().await, + PurgeStore::Data(store) => store.purge_store().await, PurgeStore::Blobs { store, blob_store } => { store.purge_blobs(blob_store.clone()).await } - PurgeStore::Lookup(store) => store.purge_expired().await, + PurgeStore::Lookup(store) => store.purge_lookup_store().await, }; if let Err(err) = result { @@ -85,7 +85,7 @@ impl PurgeSchedule { impl Display for PurgeStore { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - PurgeStore::Bitmaps(_) => write!(f, "bitmaps"), + PurgeStore::Data(_) => write!(f, "bitmaps"), PurgeStore::Blobs { .. } => write!(f, "blobs"), PurgeStore::Lookup(_) => write!(f, "expired keys"), } diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index f9837213..c670cf79 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" resolver = "2" [dependencies] -rustls = { version = "0.22", features = ["tls12"]} +rustls = { version = "0.22", default-features = false, features = ["tls12"]} rustls-pemfile = "2.0" rustls-pki-types = { version = "1" } tokio = { version = "1.23", features = ["net", "macros"] } @@ -32,7 +32,7 @@ base64 = "0.21" serde_json = "1.0" rcgen = "0.12" reqwest = { version = "0.11", default-features = false, features = ["rustls-tls-webpki-roots"]} -x509-parser = "0.15.0" +x509-parser = "0.16.0" pem = "3.0" parking_lot = "0.12" arc-swap = "1.6.0" diff --git a/resources/config/smtp/report.toml b/resources/config/smtp/report.toml index 21fc16e3..b2f27448 100644 --- a/resources/config/smtp/report.toml +++ b/resources/config/smtp/report.toml @@ -8,7 +8,7 @@ [report.analysis] addresses = ["dmarc@*", "abuse@*", "postmaster@*"] forward = true -#store = "%{BASE_PATH}%/incoming" +store = "30d" [report.dsn] from-name = "'Mail Delivery Subsystem'" diff --git a/tests/src/smtp/management/mod.rs b/tests/src/smtp/management/mod.rs index 1cd9cee9..ee4faf48 100644 --- a/tests/src/smtp/management/mod.rs +++ b/tests/src/smtp/management/mod.rs @@ -23,7 +23,7 @@ use std::time::Duration; -use reqwest::header::AUTHORIZATION; +use reqwest::{header::AUTHORIZATION, Method}; use serde::{de::DeserializeOwned, Deserialize}; pub mod queue; @@ -36,19 +36,22 @@ pub enum Response { Error { error: String, details: String }, } -pub async fn send_manage_request(query: &str) -> Result, String> { - send_manage_request_raw(query).await.map(|result| { +pub async fn send_manage_request( + method: Method, + query: &str, +) -> Result, String> { + send_manage_request_raw(method, query).await.map(|result| { serde_json::from_str::>(&result).unwrap_or_else(|err| panic!("{err}: {result}")) }) } -pub async fn send_manage_request_raw(query: &str) -> Result { +pub async fn send_manage_request_raw(method: Method, query: &str) -> Result { reqwest::Client::builder() .timeout(Duration::from_millis(500)) .danger_accept_invalid_certs(true) .build() .unwrap() - .get(format!("https://127.0.0.1:9980{query}")) + .request(method, format!("https://127.0.0.1:9980{query}")) .header(AUTHORIZATION, "Basic YWRtaW46c2VjcmV0") .send() .await @@ -69,6 +72,16 @@ impl Response { } } + pub fn try_unwrap_data(self) -> Option { + match self { + Response::Data { data } => Some(data), + Response::Error { error, .. } if error == "not-found" => None, + Response::Error { error, details } => { + panic!("Expected data, found error {error:?}: {details:?}") + } + } + } + pub fn unwrap_error(self) -> (String, String) { match self { Response::Error { error, details } => (error, details), diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index 9757aac9..77bb319b 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -30,7 +30,7 @@ use ahash::{AHashMap, HashMap, HashSet}; use directory::core::config::ConfigDirectory; use mail_auth::MX; use mail_parser::DateTime; -use reqwest::{header::AUTHORIZATION, StatusCode}; +use reqwest::{header::AUTHORIZATION, Method, StatusCode}; use store::Store; use utils::config::{if_block::IfBlock, Config, ServerProtocol}; @@ -61,9 +61,9 @@ member-of = ["superusers"] #[derive(serde::Deserialize)] #[allow(dead_code)] -struct List { - items: Vec, - total: usize, +pub(super) struct List { + pub items: Vec, + pub total: usize, } #[tokio::test] @@ -192,7 +192,7 @@ async fn manage_queue() { ); // Fetch and validate messages - let ids = send_manage_request::>("/api/queue/list") + let ids = send_manage_request::>(Method::GET, "/api/queue/messages") .await .unwrap() .unwrap_data() @@ -277,28 +277,28 @@ async fn manage_queue() { // Test list search for (query, expected_ids) in [ ( - "/api/queue/list?from=bill1@foobar.net".to_string(), + "/api/queue/messages?from=bill1@foobar.net".to_string(), vec!["a"], ), ( - "/api/queue/list?to=foobar.org".to_string(), + "/api/queue/messages?to=foobar.org".to_string(), vec!["d", "e", "f"], ), ( - "/api/queue/list?from=bill3@foobar.net&to=rcpt5@example1.com".to_string(), + "/api/queue/messages?from=bill3@foobar.net&to=rcpt5@example1.com".to_string(), vec!["c"], ), ( - format!("/api/queue/list?before={test_search}"), + format!("/api/queue/messages?before={test_search}"), vec!["a", "b"], ), ( - format!("/api/queue/list?after={test_search}"), + format!("/api/queue/messages?after={test_search}"), vec!["d", "e", "f", "c"], ), ] { let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string())); - let ids = send_manage_request::>(&query) + let ids = send_manage_request::>(Method::GET, &query) .await .unwrap() .unwrap_data() @@ -310,27 +310,24 @@ async fn manage_queue() { } // Retry delivery - assert_eq!( - send_manage_request::>(&format!( - "/api/queue/retry?id={},{}", - id_map.get("e").unwrap(), - id_map.get("f").unwrap() - )) - .await - .unwrap() - .unwrap_data(), - vec![true, true] - ); - assert_eq!( - send_manage_request::>(&format!( - "/api/queue/retry?id={}&filter=example1.org&at=2200-01-01T00:00:00Z", + for id in [id_map.get("e").unwrap(), id_map.get("f").unwrap()] { + assert!( + send_manage_request::(Method::PATCH, &format!("/api/queue/messages/{id}",)) + .await + .unwrap() + .unwrap_data(), + ); + } + assert!(send_manage_request::( + Method::PATCH, + &format!( + "/api/queue/messages/{}?filter=example1.org&at=2200-01-01T00:00:00Z", id_map.get("a").unwrap(), - )) - .await - .unwrap() - .unwrap_data(), - vec![true] - ); + ) + ) + .await + .unwrap() + .unwrap_data()); // Expect delivery to john@foobar.org tokio::time::sleep(Duration::from_millis(100)).await; @@ -384,22 +381,24 @@ async fn manage_queue() { ("c", "rcpt6@example2.com"), ("d", ""), ] { - assert_eq!( - send_manage_request::>(&format!( - "/api/queue/cancel?id={}{}{}", - id_map.get(id).unwrap(), - if !filter.is_empty() { "&filter=" } else { "" }, - filter - )) + assert!( + send_manage_request::( + Method::DELETE, + &format!( + "/api/queue/messages/{}{}{}", + id_map.get(id).unwrap(), + if !filter.is_empty() { "?filter=" } else { "" }, + filter + ) + ) .await .unwrap() .unwrap_data(), - vec![true], "failed for {id}: {filter}" ); } assert_eq!( - send_manage_request::>("/api/queue/list") + send_manage_request::>(Method::GET, "/api/queue/messages") .await .unwrap() .unwrap_data() @@ -485,14 +484,16 @@ fn assert_timestamp(timestamp: &DateTime, expected: i64, ctx: &str, message: &Me } async fn get_messages(ids: &[QueueId]) -> Vec> { - send_manage_request(&format!( - "/api/queue/status?id={}", - ids.iter() - .map(|id| id.to_string()) - .collect::>() - .join(",") - )) - .await - .unwrap() - .unwrap_data() + let mut results = Vec::with_capacity(ids.len()); + + for id in ids { + let message = + send_manage_request::(Method::GET, &format!("/api/queue/messages/{id}",)) + .await + .unwrap() + .try_unwrap_data(); + results.push(message); + } + + results } diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index beffd036..27ddac92 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -34,12 +34,16 @@ use mail_auth::{ ActionDisposition, DmarcResult, Record, }, }; +use reqwest::Method; use store::Store; use tokio::sync::mpsc; use utils::config::{if_block::IfBlock, Config, ServerProtocol}; use crate::smtp::{ - inbound::dummy_stores, management::send_manage_request, outbound::start_test_server, TestConfig, + inbound::dummy_stores, + management::{queue::List, send_manage_request}, + outbound::start_test_server, + TestConfig, }; use smtp::{ config::AggregateFrequency, @@ -141,10 +145,11 @@ async fn manage_reports() { .await; // List reports - let ids = send_manage_request::>("/admin/report/list") + let ids = send_manage_request::>(Method::GET, "/api/queue/reports") .await .unwrap() - .unwrap_data(); + .unwrap_data() + .items; assert_eq!(ids.len(), 4); let mut id_map = AHashMap::new(); let mut id_map_rev = AHashMap::new(); @@ -186,18 +191,19 @@ async fn manage_reports() { // Test list search for (query, expected_ids) in [ - ("/admin/report/list?type=dmarc", vec!["a", "b"]), - ("/admin/report/list?type=tls", vec!["c", "d"]), - ("/admin/report/list?domain=foobar.org", vec!["a", "c"]), - ("/admin/report/list?domain=foobar.net", vec!["b", "d"]), - ("/admin/report/list?domain=foobar.org&type=dmarc", vec!["a"]), - ("/admin/report/list?domain=foobar.net&type=tls", vec!["d"]), + ("/api/queue/reports?type=dmarc", vec!["a", "b"]), + ("/api/queue/reports?type=tls", vec!["c", "d"]), + ("/api/queue/reports?domain=foobar.org", vec!["a", "c"]), + ("/api/queue/reports?domain=foobar.net", vec!["b", "d"]), + ("/api/queue/reports?domain=foobar.org&type=dmarc", vec!["a"]), + ("/api/queue/reports?domain=foobar.net&type=tls", vec!["d"]), ] { let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string())); - let ids = send_manage_request::>(query) + let ids = send_manage_request::>(Method::GET, query) .await .unwrap() .unwrap_data() + .items .into_iter() .map(|id| id_map_rev.get(&id).unwrap().clone()) .collect::>(); @@ -206,23 +212,23 @@ async fn manage_reports() { // Cancel reports for id in ["a", "b"] { - assert_eq!( - send_manage_request::>(&format!( - "/admin/report/cancel?id={}", - id_map.get(id).unwrap(), - )) + assert!( + send_manage_request::( + Method::DELETE, + &format!("/api/queue/reports/{}", id_map.get(id).unwrap(),) + ) .await .unwrap() .unwrap_data(), - vec![true], "failed for {id}" ); } assert_eq!( - send_manage_request::>("/admin/report/list") + send_manage_request::>(Method::GET, "/api/queue/reports") .await .unwrap() .unwrap_data() + .items .len(), 2 ); @@ -241,8 +247,16 @@ async fn manage_reports() { } async fn get_reports(ids: &[String]) -> Vec> { - send_manage_request(&format!("/admin/report/status?id={}", ids.join(","))) - .await - .unwrap() - .unwrap_data() + let mut results = Vec::with_capacity(ids.len()); + + for id in ids { + let report = + send_manage_request::(Method::GET, &format!("/api/queue/reports/{id}",)) + .await + .unwrap() + .try_unwrap_data(); + results.push(report); + } + + results } diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index a88da95b..3fcad2ea 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -408,7 +408,7 @@ impl TestConfig for ReportConfig { addresses: vec![], forward: true, store: None, - report_id: 0.into(), + report_id: SnowflakeIdGenerator::new(), }, dkim: Report::test(), spf: Report::test(), diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs index 2bb82779..fd014c5a 100644 --- a/tests/src/smtp/reporting/analyze.rs +++ b/tests/src/smtp/reporting/analyze.rs @@ -21,25 +21,25 @@ * for more details. */ -use std::{fs, sync::Arc, time::Duration}; +use std::{sync::Arc, time::Duration}; -use crate::smtp::{ - inbound::TestQueueEvent, make_temp_dir, session::TestSession, TestConfig, TestSMTP, -}; +use crate::smtp::{inbound::TestQueueEvent, session::TestSession, TestConfig, TestSMTP}; use smtp::{ config::AddressMatch, core::{Session, SMTP}, }; +use store::{ + write::{ReportClass, ValueClass}, + IterateParams, ValueKey, +}; use utils::config::if_block::IfBlock; -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn report_analyze() { let mut core = SMTP::test(); // Create temp dir for queue let mut qr = core.init_test_queue("smtp_analyze_report_test"); - let report_dir = make_temp_dir("smtp_report_incoming", true); - let config = &mut core.session.config.rcpt; config.relay = IfBlock::new(true); let config = &mut core.session.config.data; @@ -51,10 +51,16 @@ async fn report_analyze() { AddressMatch::Equals("feedback@foobar.org".to_string()), ]; config.forward = false; - config.store = report_dir.temp_dir.clone().into(); + config.store = Duration::from_secs(1).into(); + //config.store = Duration::from_secs(86400).into(); // Create test message let core = Arc::new(core); + /*let rx_manage = crate::smtp::outbound::start_test_server( + core.clone(), + &[utils::config::ServerProtocol::Http], + );*/ + let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; @@ -84,14 +90,53 @@ async fn report_analyze() { } tokio::time::sleep(Duration::from_millis(200)).await; + //let c = tokio::time::sleep(Duration::from_secs(86400)).await; + + // Purging the database shouldn't remove the reports + qr.store.purge_store().await.unwrap(); + + // Make sure the reports are in the store let mut total_reports = 0; - for entry in fs::read_dir(&report_dir.temp_dir).unwrap() { - let path = entry.unwrap().path(); - assert_ne!(fs::metadata(&path).unwrap().len(), 0); - total_reports += 1; - } + qr.store + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), + ValueKey::from(ValueClass::Report(ReportClass::Arf { + id: u64::MAX, + expires: u64::MAX, + })), + ), + |_, _| { + total_reports += 1; + Ok(true) + }, + ) + .await + .unwrap(); assert_eq!(total_reports, total_reports_received); + // Wait one second, purge, and make sure they are gone + tokio::time::sleep(Duration::from_secs(1)).await; + qr.store.purge_store().await.unwrap(); + let mut total_reports = 0; + qr.store + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), + ValueKey::from(ValueClass::Report(ReportClass::Arf { + id: u64::MAX, + expires: u64::MAX, + })), + ), + |_, _| { + total_reports += 1; + Ok(true) + }, + ) + .await + .unwrap(); + assert_eq!(total_reports, 0); + // Test delivery to non-report addresses session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") diff --git a/tests/src/smtp/reporting/dmarc.rs b/tests/src/smtp/reporting/dmarc.rs index bb928c9f..7d387581 100644 --- a/tests/src/smtp/reporting/dmarc.rs +++ b/tests/src/smtp/reporting/dmarc.rs @@ -117,7 +117,7 @@ async fn report_dmarc() { assert_eq!(reports.len(), 1); match reports.into_iter().next().unwrap() { QueueClass::DmarcReportHeader(event) => { - core.generate_dmarc_report(event).await; + core.send_dmarc_aggregate_report(event).await; } _ => unreachable!(), } diff --git a/tests/src/smtp/reporting/tls.rs b/tests/src/smtp/reporting/tls.rs index 1e6e7f72..29672de0 100644 --- a/tests/src/smtp/reporting/tls.rs +++ b/tests/src/smtp/reporting/tls.rs @@ -85,7 +85,7 @@ async fn report_tls() { for (policy, rt) in [ ( - smtp::reporting::PolicyType::None, + smtp::reporting::PolicyType::None, // Quota limited at 1532 bytes, this should not be included in the report. ResultType::CertificateExpired, ), ( @@ -101,7 +101,7 @@ async fn report_tls() { ResultType::StsPolicyInvalid, ), ( - smtp::reporting::PolicyType::Sts(None), // Quota limited at 1532 bytes, this should not be included in the report. + smtp::reporting::PolicyType::Sts(None), ResultType::StsWebpkiInvalid, ), ] { @@ -128,8 +128,7 @@ async fn report_tls() { _ => unreachable!(), } } - core.generate_tls_report(tls_reports.first().unwrap().domain.clone(), tls_reports) - .await; + core.send_tls_aggregate_report(tls_reports).await; // Expect report let message = qr.expect_message().await; @@ -167,10 +166,10 @@ async fn report_tls() { } PolicyType::Sts => { seen[1] = true; - assert_eq!(policy.summary.total_failure, 2); + assert_eq!(policy.summary.total_failure, 3); assert_eq!(policy.summary.total_success, 0); assert_eq!(policy.policy.policy_domain, "foobar.org"); - assert_eq!(policy.failure_details.len(), 2); + assert_eq!(policy.failure_details.len(), 3); assert!(policy .failure_details .iter() @@ -182,14 +181,14 @@ async fn report_tls() { } PolicyType::NoPolicyFound => { seen[2] = true; - assert_eq!(policy.summary.total_failure, 1); + assert_eq!(policy.summary.total_failure, 0); assert_eq!(policy.summary.total_success, 2); assert_eq!(policy.policy.policy_domain, "foobar.org"); - assert_eq!(policy.failure_details.len(), 1); - assert_eq!( + assert_eq!(policy.failure_details.len(), 0); + /*assert_eq!( policy.failure_details.first().unwrap().result_type, ResultType::CertificateExpired - ); + );*/ } PolicyType::Other => unreachable!(), } @@ -218,8 +217,7 @@ async fn report_tls() { assert_eq!(reports.len(), 1); match reports.into_iter().next().unwrap() { QueueClass::TlsReportHeader(event) => { - core.generate_tls_report(event.domain.clone(), vec![event]) - .await; + core.send_tls_aggregate_report(vec![event]).await; } _ => unreachable!(), } diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index fb5ba695..7cd986cb 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -57,7 +57,7 @@ pub async fn lookup_tests() { .key_set(key.clone(), "world".to_string().into_bytes(), None) .await .unwrap(); - store.purge_expired().await.unwrap(); + store.purge_lookup_store().await.unwrap(); assert_eq!( store.key_get::(key.clone()).await.unwrap(), Some("world".to_string()) @@ -75,7 +75,7 @@ pub async fn lookup_tests() { tokio::time::sleep(tokio::time::Duration::from_secs(2)).await; assert_eq!(None, store.key_get::(key.clone()).await.unwrap()); - store.purge_expired().await.unwrap(); + store.purge_lookup_store().await.unwrap(); if let LookupStore::Store(store) = &store { store.assert_is_empty(store.clone().into()).await; } @@ -106,7 +106,7 @@ pub async fn lookup_tests() { .unwrap(); assert_eq!(1, store.counter_get(key.clone()).await.unwrap()); tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - store.purge_expired().await.unwrap(); + store.purge_lookup_store().await.unwrap(); assert_eq!(0, store.counter_get(key.clone()).await.unwrap()); // Test rate limiter @@ -127,7 +127,7 @@ pub async fn lookup_tests() { .unwrap() .is_none()); tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; - store.purge_expired().await.unwrap(); + store.purge_lookup_store().await.unwrap(); if let LookupStore::Store(store) = &store { store.assert_is_empty(store.clone().into()).await; }