diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..94f480d --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf \ No newline at end of file diff --git a/.gitignore b/.gitignore index 104c1fd..8ee9506 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,12 @@ target/ .idea logs -TODO .log .txt net-guardia/static/web +*.mmdb +node_modules/ +.next/ + +*.rules +*.db diff --git a/Cargo.lock b/Cargo.lock index b2928a5..3438e2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -301,6 +301,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anyhow" version = "1.0.100" @@ -535,6 +544,12 @@ dependencies = [ "alloc-stdlib", ] +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + [[package]] name = "byteorder" version = "1.5.0" @@ -621,6 +636,19 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-link", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -632,6 +660,15 @@ dependencies = [ "libloading", ] +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + [[package]] name = "common" version = "0.1.0" @@ -662,6 +699,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -891,6 +934,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "filetime" version = "0.2.27" @@ -936,6 +991,33 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" +[[package]] +name = "foreign-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" +dependencies = [ + "foreign-types-macros", + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-macros" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5c6c585bc94aaf2c7b51dd4c2ba22680844aba4c687be581871a6f518c5742" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "foreign-types-shared" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa9a19cbb55df58761df49b23516a86d432839add4af60fc256da840f66ed35b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -1135,6 +1217,15 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "http" version = "0.2.12" @@ -1168,6 +1259,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -1307,6 +1422,15 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "imapprotolens" +version = "0.16.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4436252f26379481b02b023a59530310d17e1d68cee692416b162a44aa68db9" +dependencies = [ + "nom 7.1.3", +] + [[package]] name = "impl-more" version = "0.1.9" @@ -1391,6 +1515,16 @@ dependencies = [ "libc", ] +[[package]] +name = "js-sys" +version = "0.3.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "kstring" version = "2.0.2" @@ -1457,6 +1591,17 @@ dependencies = [ "redox_syscall 0.7.0", ] +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libxdp-sys" version = "0.2.3+1.5.6" @@ -1704,6 +1849,21 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "net-guardia" version = "0.1.0" @@ -1715,6 +1875,7 @@ dependencies = [ "aya", "aya-log", "cargo_metadata", + "chrono", "common", "crossbeam", "dotenvy", @@ -1726,7 +1887,11 @@ dependencies = [ "maxminddb", "mime_guess", "network-types", + "ort", + "ort-tract", "parking_lot", + "protolens", + "rusqlite", "rust-embed", "serde", "serde_json", @@ -1740,6 +1905,7 @@ dependencies = [ "tracing-subscriber", "tract-onnx", "url", + "vectorscan-rs", "xsk-rs", ] @@ -1901,6 +2067,35 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray 0.17.2", + "ort-sys", + "smallvec", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" + +[[package]] +name = "ort-tract" +version = "0.3.0+0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cf5d858eff9d54a3a8935eff574c01a2962cb1ae7dc2dc0f531f75cafb8b17e" +dependencies = [ + "ort-sys", + "parking_lot", + "tract-onnx", +] + [[package]] name = "parking_lot" version = "0.12.5" @@ -1985,6 +2180,48 @@ dependencies = [ "sha2", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn 2.0.98", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project-lite" version = "0.2.16" @@ -2107,6 +2344,22 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "protolens" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c040ea146711a09a3e22158263859f189232b235626a922037a5dccab6926f6a" +dependencies = [ + "byteorder", + "futures", + "futures-util", + "imapprotolens", + "libc", + "memchr", + "nom 7.1.3", + "phf", +] + [[package]] name = "quote" version = "1.0.43" @@ -2250,6 +2503,20 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rust-embed" version = "8.7.2" @@ -2481,6 +2748,12 @@ dependencies = [ "libc", ] +[[package]] +name = "siphasher" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" + [[package]] name = "slab" version = "0.4.9" @@ -2879,9 +3152,9 @@ dependencies = [ [[package]] name = "tract-core" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d72bdfb1d8809fc16b7e3496c8a3a31e8c55eeec8f648f2715d87bd25e9db1c" +checksum = "b65d67f5190132365dda73fe215bfc5e01b031e8cbfbea9d486bb5b0dbba3545" dependencies = [ "anyhow", "anymap3", @@ -2892,7 +3165,7 @@ dependencies = [ "lazy_static", "log", "maplit", - "ndarray", + "ndarray 0.16.1", "num-complex", "num-integer", "num-traits", @@ -2905,9 +3178,9 @@ dependencies = [ [[package]] name = "tract-data" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb9833e90b72a7a8e7abc517e79a90c1463d88550531deaabd4b5dbf706a091b" +checksum = "73cd7fda1e5e8b854ea3abdd09126a87fc4af81e6d1e29ec1710a8a4abf4f13a" dependencies = [ "anyhow", "downcast-rs", @@ -2918,7 +3191,7 @@ dependencies = [ "lazy_static", "libm", "maplit", - "ndarray", + "ndarray 0.16.1", "nom 8.0.0", "nom-language", "num-integer", @@ -2931,9 +3204,9 @@ dependencies = [ [[package]] name = "tract-hir" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe98f1a0fe9d7bcc39a64258729940da99a26e0ecac8d474d17cfd15f9e4ecf" +checksum = "554df991b647dba8af0547ee5838b6912ed20b424f2adda0ea0b7faf8db1b151" dependencies = [ "derive-new", "log", @@ -2942,9 +3215,9 @@ dependencies = [ [[package]] name = "tract-linalg" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09562926176740991a4e74ada093ed4edc155a585c45e2dd6fa13994a89f04f" +checksum = "e72097a89cc4e7c5f1bc4f854b9294dd30fa6f6d8f7f409c556953b49078c94f" dependencies = [ "byteorder", "cc", @@ -2970,9 +3243,9 @@ dependencies = [ [[package]] name = "tract-nnef" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "160625a1b79132698ac292ff555c575a567a2c27cd2371c53c13225cbda8d1de" +checksum = "45b3755dd0948111b407085d11033ba218cb85b85ce8d795cec2b8353db552ea" dependencies = [ "byteorder", "flate2", @@ -2990,9 +3263,9 @@ dependencies = [ [[package]] name = "tract-onnx" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96a3bf1d24b5ca9e4371dac498c6c3cb47d1e4eb7fede41a5750e611da45274" +checksum = "ac23ad1d2d5da3256ae1a78757b1072a8a3fac2a4b28d27cfb561c5942ec2701" dependencies = [ "bytes", "derive-new", @@ -3008,9 +3281,9 @@ dependencies = [ [[package]] name = "tract-onnx-opl" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1e8a95ae93fdc53143586ec00d816c2831c6958a8a59cc4122ca5b8dba6070" +checksum = "87561bf0b84f74a124afc0f1997682728da6cd821083511e0357432954fd24f6" dependencies = [ "getrandom 0.2.17", "log", @@ -3128,6 +3401,36 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "vectorscan-rs" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "657b43a26c41ea3e5b0de95240a518a011561d204b83f8ea213fddf2cc224fb0" +dependencies = [ + "bitflags", + "foreign-types", + "libc", + "thiserror 1.0.69", + "vectorscan-rs-sys", +] + +[[package]] +name = "vectorscan-rs-sys" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "195d07e20b78923c2e8542f95a9bfd31d0cacf19f9134181c292b84055c5c7c1" +dependencies = [ + "cmake", + "flate2", + "tar", +] + [[package]] name = "version_check" version = "0.9.5" @@ -3159,6 +3462,51 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasm-bindgen" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.98", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +dependencies = [ + "unicode-ident", +] + [[package]] name = "which" version = "8.0.0" diff --git a/TODO b/TODO new file mode 100644 index 0000000..687e239 --- /dev/null +++ b/TODO @@ -0,0 +1,7 @@ +1. 加入 .csv 的紀錄資料夾，實現 .csv 過一天後自動歸檔成 xxx-yyyy-oo-zz.csv +2. will change tract-onnx engine to ort-tract engine +3. 改善推論效能及速度，在大量資料時 +4. 實現 ML + RULE 的共用 HashMap 實現共同決策結果 +5. 代碼最佳化，檢查是否除了 build.rs 以外有無 .unwarp() and eprintln() [x] +6. 完成前端 detection 頁面 +7. 使用 sqllite 實現帳號系統、白黑名單永久記錄 \ No newline at end of file diff --git a/config.toml b/config.toml index 7961902..4bc6799 100644 --- a/config.toml +++ b/config.toml @@ -3,7 +3,6 @@ ingress_ifname = "enp4s0f1" # Ingress NIC Name egress_ifname = "enp4s0f0" # Egress NIC Name geoip_db_name = "GeoLite2-City.mmdb" deep_autoencoder_name = "deep_autoencoder.onnx" -classifier_name = "classifier.onnx" models_config_name = "inference_config.json" combined_queue_count = 8 # NIC Combined Queue Count (ethtool -l ) channel_size = 4096 @@ -19,8 +18,10 @@ refresh_interval = 5 # Statistics Refresh Time max_concurrent_flows = 10000 # max_flows: track up to 10000 concurrent flows min_packets_for_inference = 5 # min_packets: minimum 10 packets per flow for inference inference_interval_secs = 5 # interval_secs: run inference every 5 seconds +min_signature_matches = 3 aggregator_window_secs = 30 inference_batch_size = 200 +flow_timeout_us = 60_000_000 traffic_logging_mode = true # When true, disables ML inference and records all ingress/egress packets to CSV traffic_log_csv_path = "traffic_log.csv" # Output CSV file path for traffic logging mode \ No newline at end of file diff --git a/models/classifier.onnx b/models/classifier.onnx deleted file mode 100644 index 43b003c..0000000 Binary files a/models/classifier.onnx and /dev/null differ diff --git a/models/deep_autoencoder.onnx b/models/deep_autoencoder.onnx deleted file mode 100644 index b5ffc06..0000000 Binary files a/models/deep_autoencoder.onnx and /dev/null differ diff --git a/models/full_config.json b/models/full_config.json deleted file mode 100644 index b590d5d..0000000 --- a/models/full_config.json +++ /dev/null @@ -1,289 +0,0 @@ -{ - "created_at": "2026-03-05T14:04:08.736531", - "framework": "PyTorch", - "model": { - "deep_autoencoder": { - "file": "deep_autoencoder.onnx", - "input_dim": 27, - "encoding_dim": 16, - "ae_feature_names": [ - "flow_duration", - "fwd_packets", - "bwd_packets", - "fwd_bytes", - "bwd_bytes", - "flow_bytes_per_sec", - "flow_pkts_per_sec", - "fwd_win_bytes", - "bwd_win_bytes", - "fwd_pkt_len_mean", - "bwd_pkt_len_mean", - "fwd_iat_mean", - "bwd_iat_mean", - "flow_iat_mean", - "pkt_len_mean", - "dst_port", - "protocol", - "psh_flag_cnt", - "ack_flag_cnt", - "syn_flag_cnt", - "fin_flag_cnt", - "rst_flag_cnt", - "pkt_len_std", - "fwd_pkt_len_std", - "bwd_pkt_len_std", - "fwd_seg_size_min", - "fwd_act_data_pkts" - ], - "ae_threshold": 0.17166012525558472 - }, - "classifier": { - "file": "classifier.onnx", - "type": "ResNet MLP", - "n_features": 28, - "n_classes": 5, - "classifier_feature_names": [ - "flow_duration", - "fwd_packets", - "bwd_packets", - "fwd_bytes", - "bwd_bytes", - "flow_bytes_per_sec", - "flow_pkts_per_sec", - "fwd_win_bytes", - "bwd_win_bytes", - "fwd_pkt_len_mean", - "bwd_pkt_len_mean", - "fwd_iat_mean", - "bwd_iat_mean", - "flow_iat_mean", - "pkt_len_mean", - "dst_port", - "protocol", - "psh_flag_cnt", - "ack_flag_cnt", - "syn_flag_cnt", - "fin_flag_cnt", - "rst_flag_cnt", - "pkt_len_std", - "fwd_pkt_len_std", - "bwd_pkt_len_std", - "fwd_seg_size_min", - "fwd_act_data_pkts", - "ae_anomaly_score" - ] - } - }, - "preprocessing": { - "ae_clip_params": { - "flow_duration": { - "lower": 1.0, - "upper": 118485085.19000001 - }, - "fwd_packets": { - "lower": 0.0, - "upper": 54.0 - }, - "bwd_packets": { - "lower": 0.0, - "upper": 82.0 - }, - "fwd_bytes": { - "lower": 0.0, - "upper": 6960.0 - }, - "bwd_bytes": { - "lower": 0.0, - "upper": 89558.6950000003 - }, - "flow_bytes_per_sec": { - "lower": 0.0, - "upper": 12500000.0 - }, - "flow_pkts_per_sec": { - "lower": 0.0226829350855, - "upper": 2000000.0 - }, - "fwd_win_bytes": { - "lower": -1.0, - "upper": 65535.0 - }, - "bwd_win_bytes": { - "lower": -1.0, - "upper": 64000.0 - }, - "fwd_pkt_len_mean": { - "lower": 0.0, - "upper": 744.7272727272729 - }, - "bwd_pkt_len_mean": { - "lower": 0.0, - "upper": 1393.9601153594992 - }, - "fwd_iat_mean": { - "lower": 0.0, - "upper": 86366722.035 - }, - "bwd_iat_mean": { - "lower": 0.0, - "upper": 21700000.0 - }, - "flow_iat_mean": { - "lower": 1.0, - "upper": 86353400.64500001 - }, - "pkt_len_mean": { - "lower": 0.0, - "upper": 945.0294113974604 - }, - "dst_port": { - "lower": 0.0, - "upper": 64499.0 - }, - "protocol": { - "lower": 0.0, - "upper": 17.0 - }, - "psh_flag_cnt": { - "lower": 0.0, - "upper": 240.0 - }, - "ack_flag_cnt": { - "lower": 0.0, - "upper": 655.0 - }, - "syn_flag_cnt": { - "lower": 0.0, - "upper": 4.0 - }, - "fin_flag_cnt": { - "lower": 0.0, - "upper": 2.0 - }, - "rst_flag_cnt": { - "lower": 0.0, - "upper": 1.0 - }, - "pkt_len_std": { - "lower": 0.0, - "upper": 943.9221842600019 - }, - "fwd_pkt_len_std": { - "lower": 0.0, - "upper": 721.4199648708689 - }, - "bwd_pkt_len_std": { - "lower": 0.0, - "upper": 911.0272448420104 - }, - "fwd_seg_size_min": { - "lower": 0.0, - "upper": 40.0 - }, - "fwd_act_data_pkts": { - "lower": 0.0, - "upper": 58.0 - } - }, - "ae_scaler": { - "mean": [ - 9783470.902880548, - 3.6532605393470496, - 3.2219949045990472, - 311.35281195367173, - 1407.508354811673, - 298997.28705051675, - 32378.36735569276, - 6158.3256759444475, - 6125.023325143747, - 58.1683929498305, - 149.72168204849987, - 2991407.86700246, - 543337.1970583026, - 2727511.340181723, - 105.03829359866097, - 13463.473246959382, - 8.920585011866685, - 5.457814320208867, - 12.214631683524651, - 0.4885976379312313, - 0.246598391240437, - 0.08478201266637703, - 151.71809799252043, - 76.89649795414182, - 143.21014925828936, - 21.447338538177014, - 3.2607463735155715 - ], - "std": [ - 27895851.239715174, - 6.4263677035863696, - 8.326275195918974, - 739.2548064174216, - 7898.711708881367, - 1187804.3239195002, - 190280.1548511611, - 15556.664474188405, - 17012.235322987635, - 98.89458775902833, - 242.79522892109904, - 13146157.964503227, - 2242915.166631539, - 12878527.145353919, - 151.95316919115754, - 21264.915159327567, - 4.943209145195101, - 23.02750953458606, - 57.0633472601567, - 1.2662749497077417, - 0.4553262784517569, - 0.278557037237646, - 211.1373807979287, - 143.47790203327, - 228.24531573376805, - 9.323458649060111, - 7.228913366949205 - ], - "feature_names": [ - "flow_duration", - "fwd_packets", - "bwd_packets", - "fwd_bytes", - "bwd_bytes", - "flow_bytes_per_sec", - "flow_pkts_per_sec", - "fwd_win_bytes", - "bwd_win_bytes", - "fwd_pkt_len_mean", - "bwd_pkt_len_mean", - "fwd_iat_mean", - "bwd_iat_mean", - "flow_iat_mean", - "pkt_len_mean", - "dst_port", - "protocol", - "psh_flag_cnt", - "ack_flag_cnt", - "syn_flag_cnt", - "fin_flag_cnt", - "rst_flag_cnt", - "pkt_len_std", - "fwd_pkt_len_std", - "bwd_pkt_len_std", - "fwd_seg_size_min", - "fwd_act_data_pkts" - ] - }, - "post_scaling_clip": { - "min": -5.0, - "max": 5.0 - } - }, - "attack_labels": { - "0": "Brute Force", - "1": "DDoS", - "2": "DoS", - "3": "Exploitation", - "4": "Reconnaissance" - } -} \ No newline at end of file diff --git a/models/inference_config.json b/models/inference_config.json deleted file mode 100644 index 1d42207..0000000 --- a/models/inference_config.json +++ /dev/null @@ -1,239 +0,0 @@ -{ - "ae_feature_names": [ - "flow_duration", - "fwd_packets", - "bwd_packets", - "fwd_bytes", - "bwd_bytes", - "flow_bytes_per_sec", - "flow_pkts_per_sec", - "fwd_win_bytes", - "bwd_win_bytes", - "fwd_pkt_len_mean", - "bwd_pkt_len_mean", - "fwd_iat_mean", - "bwd_iat_mean", - "flow_iat_mean", - "pkt_len_mean", - "dst_port", - "protocol", - "psh_flag_cnt", - "ack_flag_cnt", - "syn_flag_cnt", - "fin_flag_cnt", - "rst_flag_cnt", - "pkt_len_std", - "fwd_pkt_len_std", - "bwd_pkt_len_std", - "fwd_seg_size_min", - "fwd_act_data_pkts" - ], - "ae_clip_params": { - "flow_duration": { - "lower": 1.0, - "upper": 118485085.19000001 - }, - "fwd_packets": { - "lower": 0.0, - "upper": 54.0 - }, - "bwd_packets": { - "lower": 0.0, - "upper": 82.0 - }, - "fwd_bytes": { - "lower": 0.0, - "upper": 6960.0 - }, - "bwd_bytes": { - "lower": 0.0, - "upper": 89558.6950000003 - }, - "flow_bytes_per_sec": { - "lower": 0.0, - "upper": 12500000.0 - }, - "flow_pkts_per_sec": { - "lower": 0.0226829350855, - "upper": 2000000.0 - }, - "fwd_win_bytes": { - "lower": -1.0, - "upper": 65535.0 - }, - "bwd_win_bytes": { - "lower": -1.0, - "upper": 64000.0 - }, - "fwd_pkt_len_mean": { - "lower": 0.0, - "upper": 744.7272727272729 - }, - "bwd_pkt_len_mean": { - "lower": 0.0, - "upper": 1393.9601153594992 - }, - "fwd_iat_mean": { - "lower": 0.0, - "upper": 86366722.035 - }, - "bwd_iat_mean": { - "lower": 0.0, - "upper": 21700000.0 - }, - "flow_iat_mean": { - "lower": 1.0, - "upper": 86353400.64500001 - }, - "pkt_len_mean": { - "lower": 0.0, - "upper": 945.0294113974604 - }, - "dst_port": { - "lower": 0.0, - "upper": 64499.0 - }, - "protocol": { - "lower": 0.0, - "upper": 17.0 - }, - "psh_flag_cnt": { - "lower": 0.0, - "upper": 240.0 - }, - "ack_flag_cnt": { - "lower": 0.0, - "upper": 655.0 - }, - "syn_flag_cnt": { - "lower": 0.0, - "upper": 4.0 - }, - "fin_flag_cnt": { - "lower": 0.0, - "upper": 2.0 - }, - "rst_flag_cnt": { - "lower": 0.0, - "upper": 1.0 - }, - "pkt_len_std": { - "lower": 0.0, - "upper": 943.9221842600019 - }, - "fwd_pkt_len_std": { - "lower": 0.0, - "upper": 721.4199648708689 - }, - "bwd_pkt_len_std": { - "lower": 0.0, - "upper": 911.0272448420104 - }, - "fwd_seg_size_min": { - "lower": 0.0, - "upper": 40.0 - }, - "fwd_act_data_pkts": { - "lower": 0.0, - "upper": 58.0 - } - }, - "ae_scaler_mean": [ - 9783470.902880548, - 3.6532605393470496, - 3.2219949045990472, - 311.35281195367173, - 1407.508354811673, - 298997.28705051675, - 32378.36735569276, - 6158.3256759444475, - 6125.023325143747, - 58.1683929498305, - 149.72168204849987, - 2991407.86700246, - 543337.1970583026, - 2727511.340181723, - 105.03829359866097, - 13463.473246959382, - 8.920585011866685, - 5.457814320208867, - 12.214631683524651, - 0.4885976379312313, - 0.246598391240437, - 0.08478201266637703, - 151.71809799252043, - 76.89649795414182, - 143.21014925828936, - 21.447338538177014, - 3.2607463735155715 - ], - "ae_scaler_std": [ - 27895851.239715174, - 6.4263677035863696, - 8.326275195918974, - 739.2548064174216, - 7898.711708881367, - 1187804.3239195002, - 190280.1548511611, - 15556.664474188405, - 17012.235322987635, - 98.89458775902833, - 242.79522892109904, - 13146157.964503227, - 2242915.166631539, - 12878527.145353919, - 151.95316919115754, - 21264.915159327567, - 4.943209145195101, - 23.02750953458606, - 57.0633472601567, - 1.2662749497077417, - 0.4553262784517569, - 0.278557037237646, - 211.1373807979287, - 143.47790203327, - 228.24531573376805, - 9.323458649060111, - 7.228913366949205 - ], - "ae_post_clip_min": -5.0, - "ae_post_clip_max": 5.0, - "ae_threshold": 0.17166012525558472, - "classifier_feature_names": [ - "flow_duration", - "fwd_packets", - "bwd_packets", - "fwd_bytes", - "bwd_bytes", - "flow_bytes_per_sec", - "flow_pkts_per_sec", - "fwd_win_bytes", - "bwd_win_bytes", - "fwd_pkt_len_mean", - "bwd_pkt_len_mean", - "fwd_iat_mean", - "bwd_iat_mean", - "flow_iat_mean", - "pkt_len_mean", - "dst_port", - "protocol", - "psh_flag_cnt", - "ack_flag_cnt", - "syn_flag_cnt", - "fin_flag_cnt", - "rst_flag_cnt", - "pkt_len_std", - "fwd_pkt_len_std", - "bwd_pkt_len_std", - "fwd_seg_size_min", - "fwd_act_data_pkts", - "ae_anomaly_score" - ], - "attack_labels": { - "0": "Brute Force", - "1": "DDoS", - "2": "DoS", - "3": "Exploitation", - "4": "Reconnaissance" - } -} \ No newline at end of file diff --git a/net-guardia-frontend b/net-guardia-frontend new file mode 160000 index 0000000..c7f5045 --- /dev/null +++ b/net-guardia-frontend @@ -0,0 +1 @@ +Subproject commit c7f50458778c70a124ac07d2356a37c460b82745 diff --git a/net-guardia-frontend/package-lock.json b/net-guardia-frontend/package-lock.json deleted file mode 100644 index c93e81a..0000000 --- a/net-guardia-frontend/package-lock.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "net-guardia-frontend", - "lockfileVersion": 3, - "requires": true, - "packages": {} -} diff --git a/net-guardia/Cargo.toml b/net-guardia/Cargo.toml index 1863c54..a66c575 100644 --- a/net-guardia/Cargo.toml +++ b/net-guardia/Cargo.toml @@ -35,13 +35,21 @@ xsk-rs = { workspace = true } maxminddb = "0.27.1" lru = "0.16.2" futures = "0.3.31" -tract-onnx = "0.22.0" +tract-onnx = "0.22.1" +chrono = "0.4" +vectorscan-rs = "0.0.6" +protolens = "0.2.3" +rusqlite = { version = "0.32", features = ["bundled"] } + +ort-tract = "0.3.0+0.22" +ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "alternative-backend"] } #csv = "1.4.0" #anyhow = "1.0.100" [build-dependencies] cargo_metadata = { workspace = true } dotenvy = "0.15.7" +rusqlite = { version = "0.32", features = ["bundled"] } [[bin]] name = "net-guardia" diff --git a/net-guardia/build.rs b/net-guardia/build.rs index 57b14a8..85cc70d 100644 --- a/net-guardia/build.rs +++ b/net-guardia/build.rs @@ -1,16 +1,58 @@ use std::env; use std::fs; use std::io::{BufRead as _, BufReader}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::time::SystemTime; use cargo_metadata::{Artifact, CompilerMessage, Message, Metadata, MetadataCommand, Package, Target, TargetKind}; +/// Buffer IDs — must stay in sync with app_layer/mod.rs and rule_engine.rs. +const BUF_RAW: u8 = 0; +const BUF_HTTP_URI: u8 = 1; +const BUF_HTTP_HEADER: u8 = 2; +const BUF_HTTP_CLIENT_BODY: u8 = 3; +const BUF_HTTP_SERVER_BODY: u8 = 4; +const BUF_HTTP_METHOD: u8 = 5; +const BUF_HTTP_USER_AGENT: u8 = 6; +const BUF_HTTP_HOST: u8 = 7; +const BUF_HTTP_STAT_CODE: u8 = 8; +const BUF_HTTP_COOKIE: u8 = 9; +const BUF_HTTP_RAW_URI: u8 = 10; +const BUF_TLS_SNI: u8 = 11; +const BUF_DNS_QUERY: u8 = 12; + +struct ContentMatch { + pattern: Vec, + nocase: bool, + negated: bool, + is_fast_pattern: bool, + has_distance: bool, + has_within: bool, + has_offset: bool, + has_depth: bool, + distance: i32, + within: i32, + offset: u16, + depth: u16, + /// Target buffer (0 = raw payload, 1 = http_uri, etc.) + buffer: u8, +} + +struct SigEntry { + sid: u32, + flow_dir: u8, + dports: Vec, + msg: String, + chain: Vec, + fast_pattern_idx: usize, +} + fn main() { build_ingress_ebpf(); build_egress_ebpf(); build_frontend(); + build_vectorscan_db(); } fn build_ingress_ebpf() { @@ -88,10 +130,10 @@ fn build_ingress_ebpf() { #[allow(clippy::collapsible_match)] match message.expect("valid JSON") { Message::CompilerArtifact(Artifact { - executable, - target: Target { name, .. }, - .. - }) => { + executable, + target: Target { name, .. }, + .. + }) => { if let Some(executable) = executable { executables.push((name, executable.into_std_path_buf())); } @@ -208,10 +250,10 @@ fn build_egress_ebpf() { #[allow(clippy::collapsible_match)] match message.expect("valid JSON") { Message::CompilerArtifact(Artifact { - executable, - target: Target { name, .. }, - .. - }) => { + executable, + target: Target { name, .. }, + .. + }) => { if let Some(executable) = executable { executables.push((name, executable.into_std_path_buf())); } @@ -429,3 +471,635 @@ fn copy_dir_all(src: &PathBuf, dst: &PathBuf) -> std::io::Result<()> { } Ok(()) } + + +fn build_vectorscan_db() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let out_dir = &manifest_dir.join("static").join("db"); + let artifact_dir = &manifest_dir.join("static").join("artifacts"); + let rules_dir = &manifest_dir.join("static").join("rules"); + + println!("cargo:rerun-if-changed={}", rules_dir.display()); + if let Ok(rd) = fs::read_dir(&rules_dir) { + for entry in rd.flatten() { + let p = entry.path(); + if p.extension().and_then(|e| e.to_str()) == Some("rules") { + println!("cargo:rerun-if-changed={}", p.display()); + } + } + } + + let sigs = collect_sig_entries(&rules_dir); + + let db_path = out_dir.join("rules.db"); + write_rules_db(&db_path, &sigs); + + println!("cargo:rustc-env=RULES_DB_PATH={}", out_dir.display()); + println!("cargo:rustc-env=ARTIFACTCS_PATH={}", artifact_dir.display()); + + let total_contents: usize = sigs.iter().map(|s| s.chain.len()).sum(); + println!( + "cargo:warning=NetGuardia: {} signatures ({} content entries) ready", + sigs.len(), + total_contents + ); +} + +fn collect_sig_entries(rules_dir: &PathBuf) -> Vec { + if !rules_dir.exists() { + return Vec::new(); + } + + let mut paths: Vec<_> = fs::read_dir(rules_dir) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| p.extension().and_then(|e| e.to_str()) == Some("rules")) + .collect(); + paths.sort(); + + let mut sigs = Vec::new(); + + for path in paths { + let text = match fs::read_to_string(&path) { + Ok(t) => t, + Err(_) => continue, + }; + for line in text.lines() { + let line = line.trim(); + if line.starts_with('#') || line.is_empty() { + continue; + } + if line.contains("noalert") { + continue; + } + + let chain = extract_content_chain(line); + if chain.is_empty() { + continue; + } + + let Some(fast_idx) = select_fast_pattern(&chain) else { + continue; + }; + + sigs.push(SigEntry { + sid: extract_sid(line).unwrap_or(0), + flow_dir: extract_flow_dir(line), + dports: extract_dst_ports(line), + msg: extract_msg(line).unwrap_or_default(), + chain, + fast_pattern_idx: fast_idx, + }); + } + } + + sigs +} + +/// Map a Suricata/Snort buffer keyword to its buffer ID. +/// Handles both new-style (`http.uri`) and old-style (`http_uri`) forms. +fn detect_buffer_keyword(kw: &str) -> Option { + match kw { + "http.uri" | "http_uri" => Some(BUF_HTTP_URI), + "http.header" | "http_header" => Some(BUF_HTTP_HEADER), + "http.request_body" | "http_client_body" => Some(BUF_HTTP_CLIENT_BODY), + "http.response_body"| "http_server_body" => Some(BUF_HTTP_SERVER_BODY), + "http.method" | "http_method" => Some(BUF_HTTP_METHOD), + "http.user_agent" | "http_user_agent" => Some(BUF_HTTP_USER_AGENT), + "http.host" | "http_host" => Some(BUF_HTTP_HOST), + "http.stat_code" | "http_stat_code" => Some(BUF_HTTP_STAT_CODE), + "http.cookie" | "http_cookie" => Some(BUF_HTTP_COOKIE), + "http.raw_uri" | "http_raw_uri" => Some(BUF_HTTP_RAW_URI), + "tls.sni" | "tls_sni" => Some(BUF_TLS_SNI), + "dns_query" | "dns.query" => Some(BUF_DNS_QUERY), + _ => None, + } +} + +/// Tokenise the rule options section into `(keyword, value?)` pairs, +/// splitting on `;` while respecting double-quoted strings. +fn tokenize_options(rule: &str) -> Vec { + let opts_start = rule.find('(').map(|p| p + 1).unwrap_or(0); + let opts_end = rule.rfind(')').unwrap_or(rule.len()); + let opts = &rule[opts_start..opts_end.max(opts_start)]; + + let mut tokens: Vec = Vec::new(); + let mut current = String::new(); + let mut in_quotes = false; + + for ch in opts.chars() { + match ch { + '"' => { + in_quotes = !in_quotes; + current.push(ch); + } + ';' if !in_quotes => { + let t = current.trim().to_string(); + if !t.is_empty() { + tokens.push(t); + } + current.clear(); + } + _ => current.push(ch), + } + } + let t = current.trim().to_string(); + if !t.is_empty() { + tokens.push(t); + } + tokens +} + +fn extract_content_chain(rule: &str) -> Vec { + let tokens = tokenize_options(rule); + let mut entries: Vec = Vec::new(); + let mut sticky_buffer: u8 = BUF_RAW; + let mut i = 0; + + while i < tokens.len() { + let tok = tokens[i].as_str(); + + // New-style sticky buffer keyword (e.g. `http.uri`) + if let Some(buf) = detect_buffer_keyword(tok) { + sticky_buffer = buf; + i += 1; + continue; + } + + // content: keyword + let content_rest = if let Some(r) = tok.strip_prefix("content:") { + r + } else { + i += 1; + continue; + }; + + let negated = content_rest.starts_with('!'); + let quoted = if negated { &content_rest[1..] } else { content_rest }; + if !quoted.starts_with('"') { + i += 1; + continue; + } + + // Unquote: content token already has the full quoted string because + // tokenize_options preserves quotes inside options. + let inner = unquote_content(quoted); + let Some(raw_bytes) = parse_content_bytes(&inner) else { + i += 1; + continue; + }; + if raw_bytes.contains(&0u8) || raw_bytes.len() < 2 { + i += 1; + continue; + } + + let mut entry = ContentMatch { + pattern: raw_bytes, + nocase: false, + negated, + is_fast_pattern: false, + has_distance: false, + has_within: false, + has_offset: false, + has_depth: false, + distance: 0, + within: 0, + offset: 0, + depth: 0, + buffer: sticky_buffer, + }; + + i += 1; + + // Collect modifier tokens until the next content: or new buffer keyword. + while i < tokens.len() { + let mod_tok = tokens[i].as_str(); + + // Stop at next content: keyword — it will be handled in the outer loop. + if mod_tok.starts_with("content:") { + break; + } + + // Old-style buffer keyword overrides the sticky buffer for this entry. + if let Some(buf) = detect_buffer_keyword(mod_tok) { + entry.buffer = buf; + // A new-style sticky keyword also updates the global sticky state. + if mod_tok.contains('.') { + sticky_buffer = buf; + } + i += 1; + // Stop if new-style (becomes sticky for next content). + if mod_tok.contains('.') { break; } + continue; + } + + // Apply other modifiers. + match mod_tok { + "nocase" => entry.nocase = true, + "fast_pattern" => entry.is_fast_pattern = true, + _ => { + if let Some(v) = mod_tok.strip_prefix("distance:") { + if let Ok(n) = v.trim().parse::() { + entry.distance = n; + entry.has_distance = true; + } + } else if let Some(v) = mod_tok.strip_prefix("within:") { + if let Ok(n) = v.trim().parse::() { + entry.within = n; + entry.has_within = true; + } + } else if let Some(v) = mod_tok.strip_prefix("offset:") { + if let Ok(n) = v.trim().parse::() { + entry.offset = n; + entry.has_offset = true; + } + } else if let Some(v) = mod_tok.strip_prefix("depth:") { + if let Ok(n) = v.trim().parse::() { + entry.depth = n; + entry.has_depth = true; + } + } else if mod_tok.starts_with("fast_pattern:") { + entry.is_fast_pattern = true; + } + } + } + i += 1; + } + + entries.push(entry); + } + + entries +} + +/// Strip the surrounding double-quotes from a content token like `"foo|0d0a|"`. +fn unquote_content(s: &str) -> String { + if s.starts_with('"') && s.len() >= 2 { + // Find the closing unescaped quote. + let inner = &s[1..]; + let mut out = String::new(); + let mut chars = inner.chars(); + loop { + match chars.next() { + None | Some('"') => break, + Some('\\') => { + if let Some(c) = chars.next() { + out.push('\\'); + out.push(c); + } + } + Some(c) => out.push(c), + } + } + out + } else { + s.to_string() + } +} + +/// Select the fast-pattern index for vectorscan prefilter. +/// Prefers explicit fast_pattern keyword, then the longest non-negated pattern. +fn select_fast_pattern(chain: &[ContentMatch]) -> Option { + if let Some(idx) = chain.iter().position(|e| e.is_fast_pattern && !e.negated) { + return Some(idx); + } + chain.iter() + .enumerate() + .filter(|(_, e)| !e.negated && e.pattern.len() >= 4) + .max_by_key(|(_, e)| e.pattern.len()) + .map(|(i, _)| i) +} + +/// Write all signature data to a SQLite rules.db. +/// +/// Schema: +/// patterns(id, expression BLOB, nocase) — vectorscan prefilter; id = sig index +/// signatures(id, sid, flow_dir, msg) +/// signature_ports(sig_id, dport) +/// content_entries(sig_id, entry_order, pattern BLOB, nocase, negated, +/// has_distance, has_within, has_offset, has_depth, +/// distance, within, off, depth, buffer) +fn write_rules_db(path: &Path, sigs: &[SigEntry]) { + let _ = fs::remove_file(path); + + let conn = rusqlite::Connection::open(path) + .unwrap_or_else(|e| panic!("failed to open rules.db at {path:?}: {e}")); + + conn.execute_batch(" + CREATE TABLE patterns ( + id INTEGER PRIMARY KEY, + expression BLOB NOT NULL, + nocase INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE signatures ( + id INTEGER PRIMARY KEY, + sid INTEGER NOT NULL, + flow_dir INTEGER NOT NULL, + msg TEXT NOT NULL + ); + CREATE TABLE signature_ports ( + sig_id INTEGER NOT NULL, + dport INTEGER NOT NULL + ); + CREATE TABLE content_entries ( + sig_id INTEGER NOT NULL, + entry_order INTEGER NOT NULL, + pattern BLOB NOT NULL, + nocase INTEGER NOT NULL DEFAULT 0, + negated INTEGER NOT NULL DEFAULT 0, + has_distance INTEGER NOT NULL DEFAULT 0, + has_within INTEGER NOT NULL DEFAULT 0, + has_offset INTEGER NOT NULL DEFAULT 0, + has_depth INTEGER NOT NULL DEFAULT 0, + distance INTEGER NOT NULL DEFAULT 0, + within INTEGER NOT NULL DEFAULT 0, + off INTEGER NOT NULL DEFAULT 0, + depth INTEGER NOT NULL DEFAULT 0, + buffer INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (sig_id, entry_order) + ); + ").unwrap_or_else(|e| panic!("rules.db schema failed: {e}")); + + let tx = conn.unchecked_transaction() + .unwrap_or_else(|e| panic!("rules.db transaction failed: {e}")); + + for (sig_idx, sig) in sigs.iter().enumerate() { + let id = sig_idx as i64; + + let fp = &sig.chain[sig.fast_pattern_idx]; + let expression = regex_escape(&fp.pattern); + tx.execute( + "INSERT INTO patterns (id, expression, nocase) VALUES (?1, ?2, ?3)", + rusqlite::params![id, expression, fp.nocase as i64], + ).unwrap(); + + tx.execute( + "INSERT INTO signatures (id, sid, flow_dir, msg) VALUES (?1, ?2, ?3, ?4)", + rusqlite::params![id, sig.sid as i64, sig.flow_dir as i64, &sig.msg], + ).unwrap(); + + for &dport in &sig.dports { + tx.execute( + "INSERT INTO signature_ports (sig_id, dport) VALUES (?1, ?2)", + rusqlite::params![id, dport as i64], + ).unwrap(); + } + + for (order, entry) in sig.chain.iter().enumerate() { + tx.execute( + "INSERT INTO content_entries \ + (sig_id, entry_order, pattern, nocase, negated, \ + has_distance, has_within, has_offset, has_depth, \ + distance, within, off, depth, buffer) \ + VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14)", + rusqlite::params![ + id, + order as i64, + &entry.pattern, + entry.nocase as i64, + entry.negated as i64, + entry.has_distance as i64, + entry.has_within as i64, + entry.has_offset as i64, + entry.has_depth as i64, + entry.distance as i64, + entry.within as i64, + entry.offset as i64, + entry.depth as i64, + entry.buffer as i64, + ], + ).unwrap(); + } + } + + tx.commit().unwrap_or_else(|e| panic!("rules.db commit failed: {e}")); +} + +fn extract_sid(rule: &str) -> Option { + let pos = rule.find("sid:")?; + let rest = &rule[pos + 4..]; + let end = rest.find(|c: char| !c.is_ascii_digit()).unwrap_or(rest.len()); + rest[..end].parse().ok() +} + +fn extract_msg(rule: &str) -> Option { + let pos = rule.find("msg:\"")?; + let rest = &rule[pos + 5..]; + let mut out = String::new(); + let mut chars = rest.chars(); + loop { + match chars.next() { + None | Some('"') => break, + Some('\\') => { + if let Some(c) = chars.next() { + out.push(c); + } + } + Some(c) => out.push(c), + } + } + Some(out) +} + +/// Decode a Snort/Suricata content string into raw bytes. +/// Handles `|XX XX|` hex sections and `\;` / `\\` escapes. +fn parse_content_bytes(s: &str) -> Option> { + let mut out = Vec::new(); + let mut chars = s.chars(); + while let Some(c) = chars.next() { + match c { + '|' => { + let mut hex = String::new(); + loop { + match chars.next() { + Some('|') | None => break, + Some(h) => hex.push(h), + } + } + let hex = hex.replace(' ', ""); + if hex.len() % 2 != 0 { + return None; + } + for i in (0..hex.len()).step_by(2) { + out.push(u8::from_str_radix(&hex[i..i + 2], 16).ok()?); + } + } + '\\' => match chars.next() { + Some(';') => out.push(b';'), + Some('\\') => out.push(b'\\'), + Some(c) => { + out.push(b'\\'); + out.push(c as u8); + } + None => {} + }, + c => out.push(c as u8), + } + } + Some(out) +} + +/// Escape regex metacharacters so literal byte patterns work in vectorscan. +fn regex_escape(bytes: &[u8]) -> Vec { + let mut out = Vec::with_capacity(bytes.len() * 2); + for &b in bytes { + if matches!( + b, + b'.' | b'^' | b'$' | b'*' | b'+' | b'?' | b'(' | b')' | + b'[' | b']' | b'{' | b'}' | b'\\' | b'|' + ) { + out.push(b'\\'); + } + out.push(b); + } + out +} + +/// Returns flow direction: 0=any, 1=to_server (ingress), 2=to_client (egress). +fn extract_flow_dir(rule: &str) -> u8 { + let Some(pos) = rule.find("flow:") else { return 0 }; + let rest = &rule[pos + 5..]; + let seg_end = rest.find(';').unwrap_or(rest.len()); + let opts = &rest[..seg_end]; + if opts.contains("to_server") { 1 } + else if opts.contains("to_client") { 2 } + else { 0 } +} + +/// Parse destination ports from the rule header (token index 6, before the '('). +/// Returns empty Vec for "any" or negation-only groups (= match all). +/// Expands common Suricata/Snort port variables. +fn extract_dst_ports(rule: &str) -> Vec { + // Rule header ends at '(' + let header = match rule.find('(') { + Some(pos) => &rule[..pos], + None => return Vec::new(), + }; + + // Tokenise the header — we need token index 6 (0-based): + // action proto src_ip src_port direction dst_ip dst_port + let tokens: Vec<&str> = header.split_whitespace().collect(); + let port_token = match tokens.get(6) { + Some(t) => *t, + None => return Vec::new(), + }; + + expand_port_token(port_token) +} + +fn expand_port_token(token: &str) -> Vec { + match token { + // HTTP / Web + "$HTTP_PORTS" => return vec![80, 443, 8080, 8443, 8000, 8888], + "$HTTP_PORTS2" => return vec![80, 8080], + "$HTTPS_PORTS" => return vec![443, 8443], + "$FILE_DATA_PORTS" => return vec![80, 443, 8080, 8443, 110, 143], + "$PROXY_PORTS" => return vec![3128, 8080, 8118, 8888], + // Mail + "$SMTP_PORTS" => return vec![25, 587, 465], + "$IMAP_PORTS" => return vec![143, 993], + "$POP3_PORTS" => return vec![110, 995], + // File transfer / Remote + "$FTP_PORTS" => return vec![21], + "$FTP_DATA" => return vec![20], + "$SSH_PORTS" => return vec![22], + "$TELNET_PORTS" => return vec![23], + "$RDP_PORTS" => return vec![3389], + "$VNC_PORTS" => return vec![5900, 5901, 5902, 5903], + // Database + "$SQL_PORTS" => return vec![3306, 5432, 1433, 1521], + "$ORACLE_PORTS" => return vec![1521, 1526], + "$MSSQL_PORTS" => return vec![1433, 1434], + "$MYSQL_PORTS" => return vec![3306], + "$PGSQL_PORTS" => return vec![5432], + "$MONGODB_PORTS" => return vec![27017, 27018], + "$REDIS_PORTS" => return vec![6379], + "$MEMCACHED_PORTS" => return vec![11211], + "$ELASTICSEARCH_PORTS" => return vec![9200, 9300], + // DNS / Directory + "$DNS_PORTS" => return vec![53], + "$LDAP_PORTS" => return vec![389, 636, 3268, 3269], + "$KERBEROS_PORTS" => return vec![88, 464], + // VoIP / Messaging + "$SIP_PORTS" => return vec![5060, 5061], + "$IRC_PORTS" => return vec![6667, 6668, 6669, 7000], + "$JABBER_PORTS" => return vec![5222, 5223], + "$MSN_PORTS" => return vec![1863], + "$AIM_PORTS" => return vec![5190], + // Network management + "$SNMP_PORTS" => return vec![161, 162], + "$SYSLOG_PORTS" => return vec![514], + // Industrial / SCADA + "$MODBUS_PORTS" => return vec![502], + "$DNP3_PORTS" => return vec![20000], + "$ENIP_PORTS" => return vec![44818], + "$BACnet_PORTS" => return vec![47808], + "$VXLAN_PORTS" => return vec![4789], + "$TEREDO_PORTS" => return vec![3544], + // Shellcode / generic catch-alls + // Suricata default: !80 — we can't enumerate the complement, treat as any + "$SHELLCODE_PORTS" => return Vec::new(), + // Wildcard + "any" | "!any" => return Vec::new(), + _ => {} + } + + let token = token; // keep the &str binding + + // Any remaining unknown $VAR — treat as any (no port filter) to avoid + // infinite recursion in the group-expansion loop below. + if token.starts_with('$') { + return Vec::new(); + } + + // Negation-only → treat as any (we can't enumerate the complement) + if token.starts_with('!') && !token.starts_with("![") { + return Vec::new(); + } + + // Port group: [80,443,!8080] — strip outer brackets + let inner = if token.starts_with('[') && token.ends_with(']') { + &token[1..token.len() - 1] + } else { + token + }; + + let mut ports = Vec::new(); + let mut has_negation_only = true; + for part in inner.split(',') { + let part = part.trim(); + if part.is_empty() { continue; } + if part.starts_with('!') { + // negated entry — skip but don't block the group + continue; + } + has_negation_only = false; + // Recurse only for variable names (start with '$') or nested groups + if part.starts_with('$') || part.starts_with('[') { + ports.extend(expand_port_token(part)); + continue; + } + // Port range: 1024:2048 + if let Some(colon) = part.find(':') { + let lo: u16 = part[..colon].parse().unwrap_or(0); + let hi: u16 = part[colon + 1..].parse().unwrap_or(65535); + // Don't expand huge ranges — treat as any + if hi.saturating_sub(lo) > 1024 { + return Vec::new(); + } + for p in lo..=hi { ports.push(p); } + continue; + } + if let Ok(p) = part.parse::() { + ports.push(p); + } + } + + // If the group contained only negations, fall back to any + if has_negation_only && ports.is_empty() { + return Vec::new(); + } + + ports +} \ No newline at end of file diff --git a/net-guardia/src/core/ebpf/access_control.rs b/net-guardia/src/core/ebpf/access_control.rs index c485c0a..73df7d1 100644 --- a/net-guardia/src/core/ebpf/access_control.rs +++ b/net-guardia/src/core/ebpf/access_control.rs @@ -169,7 +169,7 @@ impl MapWrapper { Err(EbpfError::RuleReachLimit)?; } new_ports.copy_from_slice(&ports); - new_ports[index.unwrap()] = port; + new_ports[index.expect("index is Some; None returns early above")] = port; } else { new_ports[0] = port; } diff --git a/net-guardia/src/core/ebpf/mod.rs b/net-guardia/src/core/ebpf/mod.rs index 6f3066a..27f1a87 100644 --- a/net-guardia/src/core/ebpf/mod.rs +++ b/net-guardia/src/core/ebpf/mod.rs @@ -17,7 +17,7 @@ use crate::core::ebpf::xsk_manager::XskManager; use crate::core::infrastructure::app_config::AppConfig; use crate::model::error::system::SystemError; use crate::model::error::Error; -use crate::ml::engine::Engine; +use crate::detection::ml::engine::Engine; pub struct EbpfServices { pub xsk_manager: Arc, diff --git a/net-guardia/src/core/ebpf/statistics.rs b/net-guardia/src/core/ebpf/statistics.rs index f760df3..8d37afd 100644 --- a/net-guardia/src/core/ebpf/statistics.rs +++ b/net-guardia/src/core/ebpf/statistics.rs @@ -141,7 +141,7 @@ impl Statistics { let boot_time = self.boot_time; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .unwrap() + .expect("system time is after UNIX_EPOCH") .as_nanos() as u64; for ((_, _, time_type), map) in self.ipv4_maps.iter() { map.write().await.cleanup(boot_time, now, time_type.duration()) @@ -156,14 +156,16 @@ impl Statistics { direction: Direction, flow_direction: FlowDirection, time_type: TimeType, - ) -> HashMap { - let flow_data = self + ) -> Result, EbpfError> { + let map = self .ipv4_maps .get(&(direction, flow_direction, time_type)) - .unwrap() - .write() - .await - .get_map(); + .ok_or_else(|| EbpfError::FlowMapKeyMissing( + format!("{:?}", direction), + format!("{:?}", flow_direction), + format!("{:?}", time_type), + ))?; + let flow_data = map.write().await.get_map(); if let Some(ref geo_ip) = self.geo_ip { let futures: Vec<_> = flow_data @@ -177,12 +179,12 @@ impl Statistics { } }) .collect(); - join_all(futures).await.into_iter().collect() + Ok(join_all(futures).await.into_iter().collect()) } else { - flow_data + Ok(flow_data .into_iter() .map(|(addr, stats)| (addr, FlowStatsWithGeo { stats, geo: None })) - .collect() + .collect()) } } @@ -191,14 +193,16 @@ impl Statistics { direction: Direction, flow_direction: FlowDirection, time_type: TimeType, - ) -> HashMap { - let flow_data = self + ) -> Result, EbpfError> { + let map = self .ipv6_maps .get(&(direction, flow_direction, time_type)) - .unwrap() - .write() - .await - .get_map(); + .ok_or_else(|| EbpfError::FlowMapKeyMissing( + format!("{:?}", direction), + format!("{:?}", flow_direction), + format!("{:?}", time_type), + ))?; + let flow_data = map.write().await.get_map(); if let Some(ref geo_ip) = self.geo_ip { let futures: Vec<_> = flow_data @@ -212,12 +216,12 @@ impl Statistics { } }) .collect(); - join_all(futures).await.into_iter().collect() + Ok(join_all(futures).await.into_iter().collect()) } else { - flow_data + Ok(flow_data .into_iter() .map(|(addr, stats)| (addr, FlowStatsWithGeo { stats, geo: None })) - .collect() + .collect()) } } } diff --git a/net-guardia/src/core/ebpf/xsk_manager.rs b/net-guardia/src/core/ebpf/xsk_manager.rs index 1b629b6..62ddc69 100644 --- a/net-guardia/src/core/ebpf/xsk_manager.rs +++ b/net-guardia/src/core/ebpf/xsk_manager.rs @@ -17,13 +17,17 @@ use xsk_rs::config::{BindFlags, FrameSize, Interface, LibxdpFlags, QueueSize, So use xsk_rs::{CompQueue, FillQueue, FrameDesc, RxQueue, Socket, TxQueue, Umem}; use crate::core::infrastructure::app_config::AppConfig; -use crate::ml::engine::{Engine, PacketProcessor}; +use crate::detection::ml::engine::Engine; +use crate::detection::rule::rule_engine::RuleEngine; +use crate::detection::rule::stream_reassembler::StreamReassembler; use crate::model::config::Config; use crate::model::direction::Direction; use crate::model::error::ebpf::EbpfError; use crate::model::error::system::SystemError; use crate::model::error::Error; use crate::model::log::ebpf::EbpfLog; +use crate::model::log::ml::MLLog; +use crate::model::log::rule::RuleLog; pub struct XskManager { app_config: Arc, @@ -48,11 +52,22 @@ impl XskManager { }) } - pub fn run(&self, ml_engine: Option>, shutdowns: &SegQueue>) -> Result<(), Error> { + pub fn run(&self, engine: Option>, shutdowns: &SegQueue>) -> Result<(), Error> { let config = self.app_config.config.clone(); let combined_queue_count = config.combined_queue_count; - let packet_processor = ml_engine.map(|engine| Arc::new(PacketProcessor::new(engine))); + let min_sig = config.min_signature_matches; + + let rule_engine = match RuleEngine::new() { + Some(re) => { + log!(RuleLog::RulesLoaded(re.pattern_count())); + Some(Arc::new(re)) + } + None => { + log!(RuleLog::RulesDisabled); + None + } + }; for queue_id in 0..combined_queue_count { let (ingress_to_egress_tx, ingress_to_egress_rx) = bounded(config.channel_size); @@ -64,7 +79,9 @@ impl XskManager { &config.ingress_ifname, &config.egress_ifname, Direction::Ingress, - packet_processor.clone(), + engine.clone(), + rule_engine.clone(), + min_sig, )?; let egress_xsk = XskPair::new( @@ -73,7 +90,9 @@ impl XskManager { &config.egress_ifname, &config.ingress_ifname, Direction::Egress, - packet_processor.clone(), + engine.clone(), + rule_engine.clone(), + min_sig, )?; let mut xsk_map = self.xsk_map.lock(); @@ -110,8 +129,10 @@ pub struct XskPair { comp_queue: CompQueue, tx: TxQueue, rx: RxQueue, - frame_pool: Arc>>, // SegQueue - packet_processor: Option>, + frame_pool: Arc>>, + engine: Option>, + rule_engine: Option>, + min_signature_matches: u32, } impl XskPair { @@ -121,7 +142,9 @@ impl XskPair { rx_ifname: &str, tx_ifname: &str, direction: Direction, - packet_processor: Option>, + engine: Option>, + rule_engine: Option>, + min_signature_matches: u32, ) -> Result { let rx_ifname_c = CString::new(rx_ifname).map_err(|_| SystemError::UnknownError)?; @@ -176,7 +199,9 @@ impl XskPair { tx, rx, frame_pool: Arc::new(Mutex::new(pool_frames)), - packet_processor, + engine, + rule_engine, + min_signature_matches, }; Ok(xsk_pair) @@ -194,8 +219,17 @@ impl XskPair { thread::Builder::new() .name(thread_name.clone()) .spawn(move || { + // StreamReassembler is !Send (Rc inside protolens), so create it here. + let min_sig_matches = self.min_signature_matches; + let is_ingress = self.direction == Direction::Ingress; + let mut reassembler: Option = self + .rule_engine + .take() + .map(|re| StreamReassembler::new(re, min_sig_matches, is_ingress)); + let mut shutdown_rx = Some(shutdown_rx); let mut idle_count: u32 = 0; + let mut last_cleanup = std::time::Instant::now(); loop { if let Some(ref mut rx) = shutdown_rx { @@ -214,7 +248,7 @@ impl XskPair { Err(e) => log!(EbpfLog::CompQueueError(format!("{:?}", e))), } - match self.process_rx_queue(&forward_tx) { + match self.process_rx_queue(&forward_tx, &mut reassembler) { Ok(count) => total_activity += count, Err(e) => log!(EbpfLog::RXQueueError(format!("{:?}", e))), } @@ -230,6 +264,20 @@ impl XskPair { idle_count = 0; } + // Purge idle TCP flows every 60 s of wall time. + if last_cleanup.elapsed() >= Duration::from_secs(60) { + if let Some(r) = &mut reassembler { + r.cleanup_stale(Duration::from_secs(60)); + log!(RuleLog::SignatureStats( + format!("{:?}", self.direction), + r.flow_count(), + r.min_matches(), + format!("{:?}", r.match_counts()), + )); + } + last_cleanup = std::time::Instant::now(); + } + let sleep_us = match idle_count { 0..=10 => 1, 11..=100 => 10, @@ -264,7 +312,11 @@ impl XskPair { Ok(nb_completed) } - fn process_rx_queue(&mut self, forward_tx: &Sender>) -> Result { + fn process_rx_queue( + &mut self, + forward_tx: &Sender>, + reassembler: &mut Option, + ) -> Result { let mut rx_descs = vec![FrameDesc::default(); 64]; let rx_count = unsafe { self.rx.consume(&mut rx_descs) }; @@ -276,8 +328,25 @@ impl XskPair { let data = unsafe { self.umem.data(rx_desc) }; let packet_data = data.contents()[..packet_len].to_vec(); - if let Some(ref processor) = self.packet_processor { - processor.process(&packet_data, self.direction == Direction::Ingress); + if let Some(r) = &mut *reassembler { + match r.process(&packet_data) { + Ok(matches) => { + for m in matches { + log!(RuleLog::SignatureMatch( + format!("{:?}", self.direction), + m.src, + m.dst, + m.sid, + m.msg, + )); + } + } + Err(e) => log!(e), + } + } + + if let Some(ref engine) = self.engine { + engine.process_packet(&packet_data, self.direction == Direction::Ingress); } if let Err(e) = forward_tx.try_send(packet_data) { @@ -365,4 +434,4 @@ impl XskPair { Ok(nb_submitted) } -} +} \ No newline at end of file diff --git a/net-guardia/src/core/infrastructure/geoip.rs b/net-guardia/src/core/infrastructure/geoip.rs index 1810392..a1f39b1 100644 --- a/net-guardia/src/core/infrastructure/geoip.rs +++ b/net-guardia/src/core/infrastructure/geoip.rs @@ -28,7 +28,7 @@ impl GeoIpService { ) -> Result { let reader = Reader::open_readfile(db_path)?; let cache_capacity = NonZeroUsize::new(cache_size) - .unwrap_or_else(|| NonZeroUsize::new(10000).unwrap()); + .unwrap_or_else(|| NonZeroUsize::new(10000).expect("10000 is non-zero")); Ok(Self { reader: Arc::new(reader), diff --git a/net-guardia/src/core/infrastructure/health.rs b/net-guardia/src/core/infrastructure/health.rs index 59e64eb..675803e 100644 --- a/net-guardia/src/core/infrastructure/health.rs +++ b/net-guardia/src/core/infrastructure/health.rs @@ -112,7 +112,7 @@ impl SystemHealth { ) -> SystemHealthMetrics { let timestamp = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) - .unwrap() + .expect("system time is after UNIX_EPOCH") .as_secs(); let boot_time = System::boot_time(); diff --git a/net-guardia/src/core/infrastructure/ml_alert.rs b/net-guardia/src/core/infrastructure/ml_alert.rs index 2cd058d..b9b38ba 100644 --- a/net-guardia/src/core/infrastructure/ml_alert.rs +++ b/net-guardia/src/core/infrastructure/ml_alert.rs @@ -1,48 +1,9 @@ -use serde::Serialize; use tokio::sync::broadcast; use tracing::error; -use crate::model::ml_detection::DetectionResult; +use crate::model::ml_detection::{AlertMessage, DetectionResult}; -#[derive(Debug, Clone, Serialize)] -pub struct AlertMessage { - pub timestamp: u64, - pub flow_key: String, - pub src_ip: String, - pub dst_ip: String, - pub src_port: u16, - pub dst_port: u16, - pub protocol: u8, - pub is_attack: bool, - pub attack_type: Option, - pub confidence: f32, - pub ae_score: f32, -} - -impl AlertMessage { - pub fn from_detection_result(result: &DetectionResult) -> Self { - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - - Self { - timestamp, - flow_key: result.flow_key.clone(), - src_ip: result.flow_key_raw.src_ip.clone(), - dst_ip: result.flow_key_raw.dst_ip.clone(), - src_port: result.flow_key_raw.src_port, - dst_port: result.flow_key_raw.dst_port, - protocol: result.flow_key_raw.protocol, - is_attack: result.is_attack, - attack_type: result.attack_type.clone(), - confidence: result.confidence, - ae_score: result.ae_score, - } - } -} - -pub struct MLAlert { +pub struct MLAlert { broadcast_tx: broadcast::Sender, } @@ -50,9 +11,7 @@ impl MLAlert { pub fn new() -> Self { let (broadcast_tx, _) = broadcast::channel(100); - MLAlert { - broadcast_tx, - } + MLAlert { broadcast_tx } } pub fn subscribe_to_alerts(&self) -> broadcast::Receiver { @@ -77,4 +36,4 @@ impl Default for MLAlert { fn default() -> Self { Self::new() } -} \ No newline at end of file +} diff --git a/net-guardia/src/core/infrastructure/mod.rs b/net-guardia/src/core/infrastructure/mod.rs index 5f2b8c1..c84e027 100644 --- a/net-guardia/src/core/infrastructure/mod.rs +++ b/net-guardia/src/core/infrastructure/mod.rs @@ -14,15 +14,15 @@ use tokio::sync::oneshot; use crate::core::infrastructure::app_config::AppConfig; use crate::core::infrastructure::health::SystemHealth; use crate::core::infrastructure::ml_alert::MLAlert; -use crate::ml::config_loader::InferenceConfig; -use crate::ml::engine::Engine; -use crate::ml::feature_extractor::FlowFeatures; -use crate::ml::model_loader::MLModels; +use crate::detection::ml::config_loader::InferenceConfig; +use crate::detection::ml::engine::Engine; +use crate::detection::ml::feature_extractor::FlowFeatures; +use crate::detection::ml::model_loader::MLModels; use crate::model::error::misc::MiscError; -use crate::model::error::system::SystemError; use crate::model::error::Error; +use crate::model::error::system::SystemError; use crate::model::log::system::SystemLog; -use crate::ml::traffic_logger::TrafficLogger; +use crate::detection::ml::traffic_logger::TrafficLogger; pub struct AppServices { pub health: Arc, @@ -39,9 +39,11 @@ impl AppServices { let ml_models = Arc::new(MLModels::load_models(&app_config, &inference_config)?); let ml_alert = Arc::new(MLAlert::new()); + // TODO: Need Edit let traffic_logger = if app_config.traffic_logging_mode { let csv_path = app_config.traffic_log_csv_path.clone(); - let mut header = FlowFeatures::all_feature_names_owned(); + let mut header = vec!["Source IP".to_string(), "Destination IP".to_string(), "Timestamp".to_string()]; + header.extend(FlowFeatures::all_feature_names_owned()); header.push("Label".to_string()); let logger = TrafficLogger::new(&csv_path, header) .map_err(|e| MiscError::TrafficLogCreateError(csv_path.clone(), e.to_string()))?; @@ -60,9 +62,11 @@ impl AppServices { app_config.inference_batch_size, app_config.inference_interval_secs, app_config.aggregator_window_secs, + app_config.flow_timeout_us, traffic_logger, )); + Ok(Self { health: Arc::new(health), ml_alert, diff --git a/net-guardia/src/core/system.rs b/net-guardia/src/core/system.rs index 4e6d8fe..d34545d 100644 --- a/net-guardia/src/core/system.rs +++ b/net-guardia/src/core/system.rs @@ -12,7 +12,7 @@ use macros::log; use crate::core::ebpf::EbpfServices; use crate::core::infrastructure::app_config::AppConfig; use crate::core::infrastructure::AppServices; -use crate::ml::config_loader::InferenceConfig; +use crate::detection::ml::config_loader::InferenceConfig; use crate::model::error::ebpf::EbpfError; use crate::model::error::http::HttpError; use crate::model::error::misc::MiscError; @@ -73,13 +73,9 @@ impl System { log!(MLLog::ModelsLoaded( self.app_services.ml_models.get_model_info("deep_autoencoder") )); - log!(MLLog::ModelsLoaded( - self.app_services.ml_models.get_model_info("classifier") - )); log!(MLLog::ConfigLoaded { features: self.inference_config.num_ae_features(), - attacks: self.inference_config.num_attack_types() }); self.aya_log_init()?; diff --git a/net-guardia/src/ml/aggregator.rs b/net-guardia/src/detection/ml/aggregator.rs similarity index 100% rename from net-guardia/src/ml/aggregator.rs rename to net-guardia/src/detection/ml/aggregator.rs diff --git a/net-guardia/src/ml/config_loader.rs b/net-guardia/src/detection/ml/config_loader.rs similarity index 62% rename from net-guardia/src/ml/config_loader.rs rename to net-guardia/src/detection/ml/config_loader.rs index d0d161b..f70f699 100644 --- a/net-guardia/src/ml/config_loader.rs +++ b/net-guardia/src/detection/ml/config_loader.rs @@ -4,7 +4,7 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; use crate::model::error::ml::MLError; -use crate::model::ml_detection::{AENormalization, ClipParams, PrecisionLevels}; +use crate::model::ml_detection::ClipParams; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct InferenceConfig { @@ -15,15 +15,15 @@ pub struct InferenceConfig { pub ae_post_clip_min: f64, pub ae_post_clip_max: f64, pub ae_threshold: f32, - pub classifier_feature_names: Vec, - pub attack_labels: HashMap, + pub ae_threshold_method: Option, + pub window_size: usize, } impl InferenceConfig { pub fn load_file(file: &str) -> Result { - let path = PathBuf::from("models").join(file); + let path = PathBuf::from(env!("ARTIFACTCS_PATH")).join(file); let content = fs::read_to_string(&path) - .map_err(|_| MLError::ConfigLoadFailed { path: path.to_path_buf() })?; + .map_err(|_| MLError::ConfigLoadFailed { path: path.clone() })?; let config: InferenceConfig = serde_json::from_str(&content) .map_err(|e| MLError::ConfigParseFailed { reason: e.to_string() })?; Ok(config) @@ -32,16 +32,4 @@ impl InferenceConfig { pub fn num_ae_features(&self) -> usize { self.ae_feature_names.len() } - - pub fn num_classifier_features(&self) -> usize { - self.classifier_feature_names.len() - } - - pub fn num_attack_types(&self) -> usize { - self.attack_labels.len() - } - - pub fn get_attack_label(&self, id: usize) -> Option<&String> { - self.attack_labels.get(&id.to_string()) - } } \ No newline at end of file diff --git a/net-guardia/src/ml/engine.rs b/net-guardia/src/detection/ml/engine.rs similarity index 56% rename from net-guardia/src/ml/engine.rs rename to net-guardia/src/detection/ml/engine.rs index 0b67d21..971ad9e 100644 --- a/net-guardia/src/ml/engine.rs +++ b/net-guardia/src/detection/ml/engine.rs @@ -14,18 +14,20 @@ use super::model_loader::MLModels; use super::traffic_logger::TrafficLogger; use crate::core::infrastructure::ml_alert::MLAlert; +use crate::model::error::ml::MLError; use crate::model::log::ml::MLLog; use crate::model::ml_detection::{EngineStats, InferenceStats}; use crate::utils::packet_parser::parse_packet; pub struct Engine { - flow_tracker: Arc, + tracker: Arc>, inference_pipeline: Arc, aggregator: Arc>, ml_alert: Arc, min_packets: usize, batch_size: usize, inference_interval_secs: u64, + flow_timeout_us: u64, traffic_logger: Option>, } @@ -39,22 +41,26 @@ impl Engine { batch_size: usize, interval_secs: u64, window_secs: u64, + flow_timeout_us: u64, traffic_logger: Option>, ) -> Self { - let flow_tracker = Arc::new(FlowTracker::new(max_flows)); + let tracker = Arc::new(Mutex::new(FlowTracker::new(max_flows))); let inference_pipeline = Arc::new(Inference::new(models, config)); let min_detections = ((window_secs / interval_secs) / 2).max(1) as usize; let aggregator = Arc::new(Mutex::new(AttackAggregator::new(window_secs, min_detections))); + let effective_min_packets = if traffic_logger.is_some() { 1 } else { min_packets }; + Self { - flow_tracker, + tracker, inference_pipeline, aggregator, ml_alert, - min_packets, + min_packets: effective_min_packets, batch_size, inference_interval_secs: interval_secs, + flow_timeout_us, traffic_logger, } } @@ -67,8 +73,8 @@ impl Engine { shutdown_tx } - pub fn get_flow_tracker(&self) -> Arc { - self.flow_tracker.clone() + pub fn tracker(&self) -> &Arc> { + &self.tracker } async fn run_inference_loop(&self, mut shutdown_rx: oneshot::Receiver<()>) { @@ -76,15 +82,25 @@ impl Engine { loop { tokio::select! { - _ = &mut shutdown_rx => break, - _ = ticker.tick() => {} - } + _ = &mut shutdown_rx => break, + _ = ticker.tick() => {} + } - let total_flows = self.flow_tracker.flow_count(); - let all_flows = self.flow_tracker.get_flows_snapshot(); - let packet_counts: Vec = all_flows.iter().map(|f| f.packet_count()).collect(); - - let flows = self.flow_tracker.get_flows_for_inference(self.min_packets); + let Ok(mut t) = self.tracker.lock() else { + log!(MLError::TrackerLockPoisoned); + continue; + }; + let total_flows = t.flow_count(); + let packet_counts: Vec = t.get_flows_snapshot() + .iter() + .map(|f| f.packet_count()) + .collect(); + let flows = if self.traffic_logger.is_some() { + t.drain_flows(self.min_packets) + } else { + t.get_flows_for_inference(self.min_packets) + }; + drop(t); log!(MLLog::FlowStats( total_flows, @@ -94,10 +110,19 @@ impl Engine { )); if flows.is_empty() { - log!(MLLog::InferenceSkipped(format!( - "No flows with sufficient packets (total flows: {}, min packets: {})", - total_flows, self.min_packets - ))); + if self.traffic_logger.is_none() { + if let Ok(mut t) = self.tracker.lock() { + t.cleanup_old_flows(self.flow_timeout_us); + let active_ips: std::collections::HashSet = t + .get_flows_snapshot().iter() + .map(|f| f.flow_key.src_ip.clone()) + .collect(); + drop(t); + self.inference_pipeline.cleanup_buffers(&active_ips); + } else { + log!(MLError::TrackerLockPoisoned); + } + } continue; } @@ -107,18 +132,25 @@ impl Engine { let features = FlowFeatures::extract(flow, &feature_names); logger.log_row(features.to_csv_record()); } - self.flow_tracker.cleanup_old_flows(60_000_000); + if let Ok(mut t) = self.tracker.lock() { + t.cleanup_old_flows(self.flow_timeout_us); + } else { + log!(MLError::TrackerLockPoisoned); + } continue; } - let batch = &flows[..flows.len().min(self.batch_size)]; + let mut batch = flows[..flows.len().min(self.batch_size)].to_vec(); + batch.sort_by(|a, b| { + a.flow_key.src_ip.cmp(&b.flow_key.src_ip) + .then_with(|| a.start_time_us.cmp(&b.start_time_us)) + }); log!(MLLog::RunningInference(batch.len())); let start = Instant::now(); - let results = self.inference_pipeline.infer_batch(batch); + let results = self.inference_pipeline.infer_batch(&batch); let elapsed_us = start.elapsed().as_micros() as u64; - let stats = InferenceStats::from_results(&results, elapsed_us); if results.len() != batch.len() { @@ -136,9 +168,11 @@ impl Engine { if let Ok(mut aggregator) = self.aggregator.lock() { for result in &results { if result.is_attack { - let should_alert = - aggregator.should_alert(&result.flow_key_raw, result.ae_score, result.threshold); - + let should_alert = aggregator.should_alert( + &result.flow_key_raw, + result.ae_score, + result.threshold, + ); if should_alert { log!(MLLog::ThreatDetected( format!("{:?}", result.direction), @@ -147,16 +181,24 @@ impl Engine { result.confidence, result.ae_score, )); - self.ml_alert.broadcast_alert(result); } } } - aggregator.cleanup(); } - self.flow_tracker.cleanup_old_flows(60_000_000); + if let Ok(mut t) = self.tracker.lock() { + t.cleanup_old_flows(self.flow_timeout_us); + let active_ips: std::collections::HashSet = t + .get_flows_snapshot().iter() + .map(|f| f.flow_key.src_ip.clone()) + .collect(); + drop(t); + self.inference_pipeline.cleanup_buffers(&active_ips); + } else { + log!(MLError::TrackerLockPoisoned); + } } } @@ -164,35 +206,24 @@ impl Engine { match parse_packet(packet_data) { Some((packet_info, payload_start)) => { let payload = packet_data.get(payload_start..).unwrap_or(&[]); - self.flow_tracker.process_packet(packet_info, is_ingress, payload); + if let Ok(mut t) = self.tracker.lock() { + t.process_packet(packet_info, is_ingress, payload); + } else { + log!(MLError::TrackerLockPoisoned); + } } None => log!(MLLog::ParsePacketFailed(packet_data.len())), } } pub fn get_stats(&self) -> EngineStats { - EngineStats { - active_flows: self.flow_tracker.flow_count(), - } - } -} - -pub struct PacketProcessor { - ml_engine: Arc, -} - -impl PacketProcessor { - pub fn new(ml_engine: Arc) -> Self { - Self { ml_engine } - } - - pub fn process(&self, packet_data: &[u8], is_ingress: bool) { - self.ml_engine.process_packet(packet_data, is_ingress); - } - - pub fn process_batch(&self, packets: &[Vec], is_ingress: bool) { - for packet in packets { - self.process(packet, is_ingress); - } + let active_flows = match self.tracker.lock() { + Ok(t) => t.flow_count(), + Err(_) => { + log!(MLError::TrackerLockPoisoned); + 0 + } + }; + EngineStats { active_flows } } } diff --git a/net-guardia/src/ml/feature_extractor.rs b/net-guardia/src/detection/ml/feature_extractor.rs similarity index 91% rename from net-guardia/src/ml/feature_extractor.rs rename to net-guardia/src/detection/ml/feature_extractor.rs index 8727668..98bdf66 100644 --- a/net-guardia/src/ml/feature_extractor.rs +++ b/net-guardia/src/detection/ml/feature_extractor.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use chrono::{DateTime, Utc, TimeZone}; use super::flow_tracker::FlowData; use crate::model::ml_detection::{ClipParams, PacketData}; @@ -7,6 +8,10 @@ use crate::model::ml_detection::{ClipParams, PacketData}; pub struct FlowFeatures { pub features: Vec, pub feature_num: usize, + + pub src_ip: String, + pub dst_ip: String, + pub timestamp: u64, } impl FlowFeatures { @@ -15,17 +20,20 @@ impl FlowFeatures { let mut features = Vec::with_capacity(feature_num); for name in feature_names { - let value = Self::get_feature_by_name(flow, name.trim()); - features.push(value); + features.push(Self::get_feature_by_name(flow, name.trim())); + } + Self { + features, + feature_num, + src_ip: flow.flow_key.src_ip.clone(), + dst_ip: flow.flow_key.dst_ip.clone(), + timestamp: flow.start_time_us, } - - Self { features, feature_num } } fn get_feature_by_name(flow: &FlowData, feature_name: &str) -> f64 { let safe_div = |a: f64, b: f64| if b > 0.0 { a / b } else { 0.0 }; - // 1-5 let fwd_count = flow.fwd_packets.len() as f64; let bwd_count = flow.bwd_packets.len() as f64; let total_count = fwd_count + bwd_count; @@ -34,64 +42,48 @@ impl FlowFeatures { let duration_s = duration_us / 1_000_000.0; let duration_s = if duration_s > 0.0 { duration_s } else { 1e-6 }; - // 6-9 - let fwd_lengths: Vec = flow.fwd_packets.iter().map(|p| p.length as f64).collect(); + let fwd_lengths: Vec = flow.fwd_packets.iter().map(|p| p.payload_length as f64).collect(); let (fwd_max, fwd_min, fwd_mean, fwd_std) = compute_stats(&fwd_lengths); - // 10-13 - let bwd_lengths: Vec = flow.bwd_packets.iter().map(|p| p.length as f64).collect(); + let bwd_lengths: Vec = flow.bwd_packets.iter().map(|p| p.payload_length as f64).collect(); let (bwd_max, bwd_min, bwd_mean, bwd_std) = compute_stats(&bwd_lengths); - // 14-15 let total_bytes = (flow.fwd_total_bytes + flow.bwd_total_bytes) as f64; - // 16-19 let flow_iats = compute_flow_iats(&flow.fwd_packets, &flow.bwd_packets); let (flow_iat_max, flow_iat_min, flow_iat_mean, flow_iat_std) = compute_stats(&flow_iats); - // 20-24 let fwd_iats = compute_iats(&flow.fwd_packets); let fwd_iat_total: f64 = fwd_iats.iter().sum(); let (fwd_iat_max, fwd_iat_min, fwd_iat_mean, fwd_iat_std) = compute_stats(&fwd_iats); - // 25-29 let bwd_iats = compute_iats(&flow.bwd_packets); let bwd_iat_total: f64 = bwd_iats.iter().sum(); let (bwd_iat_max, bwd_iat_min, bwd_iat_mean, bwd_iat_std) = compute_stats(&bwd_iats); - // 30-37 let fwd_psh = flow.fwd_packets.iter().filter(|p| p.flags.psh).count() as f64; let bwd_psh = flow.bwd_packets.iter().filter(|p| p.flags.psh).count() as f64; let fwd_urg = flow.fwd_packets.iter().filter(|p| p.flags.urg).count() as f64; let bwd_urg = flow.bwd_packets.iter().filter(|p| p.flags.urg).count() as f64; - // 38-55 let all_lengths: Vec = flow - .fwd_packets - .iter() - .chain(flow.bwd_packets.iter()) - .map(|p| p.length as f64) + .fwd_packets.iter().chain(flow.bwd_packets.iter()) + .map(|p| p.payload_length as f64) .collect(); - let (max_len, min_len, mean_len, std_len) = compute_stats(&all_lengths); - // 56-67 let fwd_bulk = &flow.fwd_bulk_state; let bwd_bulk = &flow.bwd_bulk_state; - // 68-69 - let fwd_seg_sizes: Vec = flow + let fwd_header_sizes: Vec = flow .fwd_packets .iter() - .filter(|p| p.payload_length > 0) .map(|p| p.header_length as f64) .collect(); - // 70-73 let (active_max, active_min, active_mean, active_std) = compute_stats(&flow.active_periods.iter().map(|&x| x as f64).collect::>()); - // 74-77 let (idle_max, idle_min, idle_mean, idle_std) = compute_stats(&flow.idle_periods.iter().map(|&x| x as f64).collect::>()); @@ -171,8 +163,8 @@ impl FlowFeatures { "Subflow Bwd Bytes" => flow.bwd_total_bytes as f64, "Init_Win_bytes_forward" | "Init Fwd Win Byts" | "fwd_win_bytes" => flow.init_win_bytes_fwd as f64, "Init_Win_bytes_backward" | "Init Bwd Win Byts" | "bwd_win_bytes" => flow.init_win_bytes_bwd as f64, - "act_data_pkt_fwd" | "Fwd Act Data Pkts" | "fwd_act_data_pkts" => fwd_seg_sizes.len() as f64, - "min_seg_size_forward" | "Fwd Seg Size Min" | "fwd_seg_size_min" => fwd_seg_sizes + "act_data_pkt_fwd" | "Fwd Act Data Pkts" | "fwd_act_data_pkts" => flow.act_data_pkt_fwd as f64, + "min_seg_size_forward" | "Fwd Seg Size Min" | "fwd_seg_size_min" => fwd_header_sizes .iter() .min_by(|a, b| a.total_cmp(b)) .copied() @@ -185,7 +177,6 @@ impl FlowFeatures { "Idle Std" => idle_std, "Idle Max" => idle_max, "Idle Min" => idle_min, - _ => 0.0, } } @@ -309,7 +300,14 @@ impl FlowFeatures { } pub fn to_csv_record(&self) -> Vec { - let mut record: Vec = self.features.iter().map(|f| f.to_string()).collect(); + let ts_ms = self.timestamp / 1000; + let ts_str = match Utc.timestamp_millis_opt(ts_ms as i64) { + chrono::LocalResult::Single(dt) => dt.format("%m/%d/%Y %H:%M:%S").to_string(), + _ => ts_ms.to_string(), + }; + + let mut record = vec![self.src_ip.clone(), self.dst_ip.clone(), ts_str]; + record.extend(self.features.iter().map(|f| f.to_string())); record.push("BENIGN".to_string()); record } @@ -327,7 +325,11 @@ fn compute_stats(values: &[f64]) -> (f64, f64, f64, f64) { let max = values.iter().cloned().fold(f64::NEG_INFINITY, f64::max); let min = values.iter().cloned().fold(f64::INFINITY, f64::min); - let variance: f64 = values.iter().map(|&x| (x - mean).powi(2)).sum::() / n; + let variance: f64 = if n > 1.0 { + values.iter().map(|&x| (x - mean).powi(2)).sum::() / (n - 1.0) + } else { + 0.0 + }; let std = variance.sqrt(); (max, min, mean, std) diff --git a/net-guardia/src/ml/flow_tracker.rs b/net-guardia/src/detection/ml/flow_tracker.rs similarity index 62% rename from net-guardia/src/ml/flow_tracker.rs rename to net-guardia/src/detection/ml/flow_tracker.rs index 1e76163..76684a3 100644 --- a/net-guardia/src/ml/flow_tracker.rs +++ b/net-guardia/src/detection/ml/flow_tracker.rs @@ -1,5 +1,4 @@ use std::collections::HashMap; -use std::sync::{Arc, Mutex}; use std::time; use common::model::event::Event; @@ -34,6 +33,10 @@ pub struct FlowData { pub last_packet_time: u64, pub fwd_bulk_state: BulkState, pub bwd_bulk_state: BulkState, + pub act_data_pkt_fwd: u32, + active_start_us: u64, + fwd_fin_seen: bool, + bwd_fin_seen: bool, } impl FlowData { @@ -72,10 +75,17 @@ impl FlowData { last_packet_time: first_packet.timestamp_us(), fwd_bulk_state: BulkState::default(), bwd_bulk_state: BulkState::default(), + act_data_pkt_fwd: 0, + active_start_us: first_packet.timestamp_us(), + fwd_fin_seen: false, + bwd_fin_seen: false, } } pub fn add_packet(&mut self, packet: &Event) { + const MAX_PACKETS_PER_DIRECTION: usize = 1000; + const MAX_PERIODS: usize = 10000; + let packet_data = PacketData { timestamp_us: packet.timestamp_us(), length: packet.packet_length(), @@ -86,69 +96,86 @@ impl FlowData { if packet.tcp_flags().fin { self.fin_count += 1; + // Track per-direction FIN for proper 4-way teardown detection + if packet.is_forward() { + self.fwd_fin_seen = true; + } else { + self.bwd_fin_seen = true; + } } - if packet.tcp_flags().syn { - self.syn_count += 1; - } - if packet.tcp_flags().rst { - self.rst_count += 1; - } - if packet.tcp_flags().psh { - self.psh_count += 1; - } - if packet.tcp_flags().ack { - self.ack_count += 1; - } - if packet.tcp_flags().urg { - self.urg_count += 1; - } - if packet.tcp_flags().cwr { - self.cwe_count += 1; - } - if packet.tcp_flags().ece { - self.ece_count += 1; - } + if packet.tcp_flags().syn { self.syn_count += 1; } + if packet.tcp_flags().rst { self.rst_count += 1; } + if packet.tcp_flags().psh { self.psh_count += 1; } + if packet.tcp_flags().ack { self.ack_count += 1; } + if packet.tcp_flags().urg { self.urg_count += 1; } + if packet.tcp_flags().cwr { self.cwe_count += 1; } + if packet.tcp_flags().ece { self.ece_count += 1; } let iat = packet.timestamp_us().saturating_sub(self.last_packet_time); const IDLE_THRESHOLD_US: u64 = 1_000_000; if iat > IDLE_THRESHOLD_US { - self.idle_periods.push(iat); - } else if iat > 0 { - self.active_periods.push(iat); + // Record cumulative active period before this idle gap + let active_dur = self.last_packet_time.saturating_sub(self.active_start_us); + if active_dur > 0 && self.active_periods.len() < MAX_PERIODS { + self.active_periods.push(active_dur); + } + if self.idle_periods.len() < MAX_PERIODS { + self.idle_periods.push(iat); + } + self.active_start_us = packet.timestamp_us(); } self.last_packet_time = packet.timestamp_us(); self.last_time_us = packet.timestamp_us(); - if packet.is_forward() { - self.fwd_packets.push(packet_data.clone()); - self.fwd_total_bytes += packet.packet_length() as u64; - self.fwd_header_bytes += packet.header_length() as u64; + if packet.is_forward() && packet.payload_length() > 0 { + self.act_data_pkt_fwd += 1; + } + if packet.is_forward() { + if self.fwd_packets.len() < MAX_PACKETS_PER_DIRECTION { + self.fwd_packets.push(packet_data.clone()); + } + self.fwd_total_bytes += packet.payload_length() as u64; + self.fwd_header_bytes += packet.header_length() as u64; if self.init_win_bytes_fwd == 0 { self.init_win_bytes_fwd = packet.tcp_window_size(); } - Self::update_bulk_state(&mut self.fwd_bulk_state, &packet_data); } else { - self.bwd_packets.push(packet_data.clone()); - self.bwd_total_bytes += packet.packet_length() as u64; + if self.bwd_packets.len() < MAX_PACKETS_PER_DIRECTION { + self.bwd_packets.push(packet_data.clone()); + } + self.bwd_total_bytes += packet.payload_length() as u64; self.bwd_header_bytes += packet.header_length() as u64; - if self.init_win_bytes_bwd == 0 { self.init_win_bytes_bwd = packet.tcp_window_size(); } - Self::update_bulk_state(&mut self.bwd_bulk_state, &packet_data); } } + /// TCP flow is finished when both sides have sent FIN, or either side sent RST. + pub fn is_finished(&self) -> bool { + (self.fwd_fin_seen && self.bwd_fin_seen) || self.rst_count > 0 + } + fn update_bulk_state(bulk_state: &mut BulkState, packet: &PacketData) { - const BULK_MIN_PACKETS: u64 = 4; - const BULK_MIN_BYTES: u64 = 1000; + const BULK_IDLE_US: u64 = 1_000_000; if packet.payload_length > 0 { + // Idle break: discard helper, start fresh + if bulk_state.in_bulk + && packet.timestamp_us.saturating_sub(bulk_state.last_bulk_packet_us) > BULK_IDLE_US + { + bulk_state.in_bulk = false; + bulk_state.last_bulk_bytes = 0; + bulk_state.last_bulk_packets = 0; + bulk_state.last_bulk_start_us = 0; + bulk_state.last_bulk_packet_us = 0; + } + if !bulk_state.in_bulk { bulk_state.in_bulk = true; bulk_state.last_bulk_bytes = packet.length as u64; @@ -156,22 +183,28 @@ impl FlowData { bulk_state.last_bulk_start_us = packet.timestamp_us; bulk_state.last_bulk_packet_us = packet.timestamp_us; } else { + let prev_us = bulk_state.last_bulk_packet_us; bulk_state.last_bulk_bytes += packet.length as u64; bulk_state.last_bulk_packets += 1; bulk_state.last_bulk_packet_us = packet.timestamp_us; + + if bulk_state.last_bulk_packets == 4 { + // Threshold first reached: commit initial 4 packets + bulk_state.bulk_count += 1; + bulk_state.total_packets += 4; + bulk_state.total_bytes += bulk_state.last_bulk_bytes; + bulk_state.total_duration_us += packet.timestamp_us + .saturating_sub(bulk_state.last_bulk_start_us); + } else if bulk_state.last_bulk_packets > 4 { + // Each subsequent packet adds incrementally + bulk_state.total_packets += 1; + bulk_state.total_bytes += packet.length as u64; + bulk_state.total_duration_us += packet.timestamp_us + .saturating_sub(prev_us); + } } } else { - if bulk_state.in_bulk - && bulk_state.last_bulk_packets >= BULK_MIN_PACKETS - && bulk_state.last_bulk_bytes >= BULK_MIN_BYTES - { - bulk_state.bulk_count += 1; - bulk_state.total_bytes += bulk_state.last_bulk_bytes; - bulk_state.total_packets += bulk_state.last_bulk_packets; - bulk_state.total_duration_us += bulk_state - .last_bulk_packet_us - .saturating_sub(bulk_state.last_bulk_start_us); - } + // Zero-payload: end bulk sequence (no commit — incomplete bulks discarded) bulk_state.in_bulk = false; bulk_state.last_bulk_bytes = 0; bulk_state.last_bulk_packets = 0; @@ -189,43 +222,39 @@ impl FlowData { } } +/// 每個 thread 獨立擁有，不共享，無鎖。 +/// RSS 保證同一條 flow 永遠落在同一個 queue。 pub struct FlowTracker { - flows: Arc>>, + flows: HashMap, max_flows: usize, } impl FlowTracker { pub fn new(max_flows: usize) -> Self { Self { - flows: Arc::new(Mutex::new(HashMap::new())), + flows: HashMap::new(), max_flows, } } - pub fn process_packet(&self, mut packet: Event, is_ingress: bool, payload: &[u8]) { + pub fn process_packet(&mut self, mut packet: Event, is_ingress: bool, payload: &[u8]) { let direction = if is_ingress { Direction::Ingress } else { Direction::Egress }; - let packet_key = FlowKey::from_packet(&packet); - let proto = packet_key.protocol; - let src_port = packet_key.src_port; - let dst_port = packet_key.dst_port; - let reversed_key = packet_key.clone().reverse(); - let Ok(mut flows) = self.flows.lock() else { - return; - }; + let packet_key = FlowKey::from_packet(&packet); + let reversed_key = packet_key.clone().reverse(); // Try-both: canonical key is whichever orientation already exists in the flow table. // For new flows, identify the initiator using (in priority order): // 1. TCP SYN / SYN+ACK flags // 2. DPI: TLS ClientHello/ServerHello, HTTP request/response, DNS QR bit // 3. Best effort: use packet as-is - let (actual_key, is_forward) = if flows.contains_key(&packet_key) { + let (actual_key, is_forward) = if self.flows.contains_key(&packet_key) { (packet_key, true) - } else if flows.contains_key(&reversed_key) { + } else if self.flows.contains_key(&reversed_key) { (reversed_key, false) } else { let flags = packet.tcp_flags(); @@ -240,7 +269,7 @@ impl FlowTracker { } else if flags.syn { (packet_key, true) } else { - match detect_initiator(payload, proto, src_port, dst_port) { + match detect_initiator(payload, packet_key.protocol, packet_key.src_port, packet_key.dst_port) { Some(true) => (packet_key, true), Some(false) => (reversed_key, false), None => (packet_key, true), @@ -250,64 +279,64 @@ impl FlowTracker { packet.set_is_forward(is_forward); - // `direction` should reflect the initiator's interface. - // If this packet is backward (is_forward = false), the initiator is on the opposite side. let initiator_direction = if is_forward { direction } else { direction.flip() }; - let flow = flows + let flow = self.flows .entry(actual_key.clone()) .or_insert_with(|| FlowData::new(actual_key, &packet, initiator_direction)); flow.add_packet(&packet); - if flows.len() > self.max_flows { - if let Some(key) = flows.keys().next().cloned() { - flows.remove(&key); + if self.flows.len() > self.max_flows { + if let Some(key) = self.flows.keys().next().cloned() { + self.flows.remove(&key); } } } + pub fn drain_flows(&mut self, min_packets: usize) -> Vec { + let mut result = Vec::new(); + self.flows.retain(|_, flow| { + if flow.packet_count() >= min_packets { + result.push(flow.clone()); + false + } else { + true + } + }); + result + } + pub fn get_flows_snapshot(&self) -> Vec { - let Ok(flows) = self.flows.lock() else { - return Vec::new(); - }; - flows.values().cloned().collect() + self.flows.values().cloned().collect() } pub fn get_flows_for_inference(&self, min_packets: usize) -> Vec { - let Ok(flows) = self.flows.lock() else { - return Vec::new(); - }; - flows + self.flows .values() .filter(|flow| flow.packet_count() >= min_packets) .cloned() .collect() } - pub fn cleanup_old_flows(&self, max_age_us: u64) { + pub fn cleanup_old_flows(&mut self, max_age_us: u64) { let now = time::SystemTime::now() .duration_since(time::UNIX_EPOCH) .map(|d| d.as_micros() as u64) .unwrap_or(0); - - let Ok(mut flows) = self.flows.lock() else { - return; - }; - flows.retain(|_, flow| now.saturating_sub(flow.last_time_us) < max_age_us); + self.flows.retain(|_, flow| { + // Remove flows that have completed TCP teardown (both FIN or RST) + // or have exceeded the idle timeout + !flow.is_finished() + && now.saturating_sub(flow.last_time_us) < max_age_us + }); } pub fn flow_count(&self) -> usize { - let Ok(flows) = self.flows.lock() else { - return 0; - }; - flows.len() + self.flows.len() } } -/// Inspect payload bytes to determine which side is the flow initiator. -/// Returns Some(true) if this packet is from the initiator, Some(false) if from the responder, -/// or None if the payload gives no useful signal. fn detect_initiator(payload: &[u8], protocol: u8, src_port: u16, dst_port: u16) -> Option { if payload.is_empty() { return None; diff --git a/net-guardia/src/detection/ml/inference.rs b/net-guardia/src/detection/ml/inference.rs new file mode 100644 index 0000000..900d386 --- /dev/null +++ b/net-guardia/src/detection/ml/inference.rs @@ -0,0 +1,153 @@ +use std::collections::VecDeque; +use std::collections::HashMap; +use std::sync::Mutex; + +use macros::log; +use tract_onnx::prelude::*; + +use super::config_loader::InferenceConfig; +use super::feature_extractor::FlowFeatures; +use super::flow_tracker::FlowData; +use super::model_loader::MLModels; +use crate::model::error::ml::MLError; +use crate::model::log::ml::MLLog; +use crate::model::ml_detection::DetectionResult; + +pub struct Inference { + pub models: Arc, + pub config: Arc, + // per-src_ip sliding window buffer: src_ip -> deque of feature vectors + flow_buffers: Mutex>>>, +} + +impl Inference { + pub fn new(models: Arc, config: Arc) -> Self { + Self { + models, + config, + flow_buffers: Mutex::new(HashMap::new()), + } + } + + pub fn cleanup_buffers(&self, active_src_ips: &std::collections::HashSet) { + let Ok(mut buffers) = self.flow_buffers.lock() else { + log!(MLError::InferenceLockPoisoned); + return; + }; + buffers.retain(|ip, _| active_src_ips.contains(ip)); + } + + pub fn infer_batch(&self, flows: &[FlowData]) -> Vec { + flows.iter().filter_map(|flow| self.infer_single(flow)).collect() + } + + pub fn infer_single(&self, flow: &FlowData) -> Option { + let features = self.preprocess_ae_features(flow); + let window_size = self.config.window_size; + + // Update per-src_ip buffer + let sequence = { + let Ok(mut buffers) = self.flow_buffers.lock() else { + log!(MLError::InferenceLockPoisoned); + return None; + }; + let buf = buffers + .entry(flow.flow_key.src_ip.clone()) + .or_insert_with(VecDeque::new); + + buf.push_back(features.clone()); + if buf.len() > window_size { + buf.pop_front(); + } + + println!( + "Buffer [{}->{}]: {}/{} | contents: {:?}", + flow.flow_key.src_ip, + flow.flow_key.dst_ip, + buf.len(), + window_size, + buf.iter().map(|v| format!("{:.3}", v[0])).collect::>() + ); + if buf.len() < window_size { + return None; + } + + // Left-pad with zeros if not enough history + let mut seq: Vec> = Vec::with_capacity(window_size); + let pad_len = window_size.saturating_sub(buf.len()); + let feat_len = self.config.num_ae_features(); + for _ in 0..pad_len { + seq.push(vec![0.0f32; feat_len]); + } + for v in buf.iter() { + seq.push(v.clone()); + } + seq + }; + + // Build 3D tensor (1, window_size, features) + let feat_len = self.config.num_ae_features(); + let ae_input = tract_ndarray::Array3::from_shape_fn( + (1, window_size, feat_len), + |(_, t, f)| sequence[t][f], + ); + + let ae_score = match self.run_autoencoder(&ae_input) { + Ok(score) => score, + Err(e) => { + log!(MLLog::InferenceFailed("LSTMAutoEncoder".to_string(), e.to_string())); + return None; + } + }; + + let is_attack = ae_score >= self.config.ae_threshold; + + let flow_key = format!( + "{}:{} -> {}:{} (proto {}) [{}]", + flow.flow_key.src_ip, + flow.flow_key.src_port, + flow.flow_key.dst_ip, + flow.flow_key.dst_port, + flow.flow_key.protocol, + flow.direction + ); + + Some(DetectionResult { + flow_key, + flow_key_raw: flow.flow_key.clone(), + direction: flow.direction, + is_attack, + attack_type: if is_attack { Some("ANOMALY".to_string()) } else { None }, + confidence: ae_score, + ae_score, + threshold: self.config.ae_threshold, + }) + } + + fn preprocess_ae_features(&self, flow: &FlowData) -> Vec { + let mut features = FlowFeatures::extract(flow, &self.config.ae_feature_names); + features.winsorize(&self.config.ae_clip_params, &self.config.ae_feature_names); + features.normalize(&self.config.ae_scaler_mean, &self.config.ae_scaler_std); + features.clip(self.config.ae_post_clip_min, self.config.ae_post_clip_max); + features.features.iter().map(|&x| x as f32).collect() + } + + fn run_autoencoder(&self, input: &tract_ndarray::Array3) -> TractResult { + let result = self + .models + .deep_autoencoder + .run(tvec![input.clone().into_tensor().into()])?; + + let output = result[0] + .to_array_view::()? + .into_dimensionality::()?; + + // MSE between input and reconstructed output + let input_view = input.view(); + let diff = &input_view - &output; + let n = (self.config.window_size * self.config.num_ae_features()) as f32; + let mse = (&diff * &diff).sum() / n; + + Ok(mse) + } +} \ No newline at end of file diff --git a/net-guardia/src/ml/mod.rs b/net-guardia/src/detection/ml/mod.rs similarity index 86% rename from net-guardia/src/ml/mod.rs rename to net-guardia/src/detection/ml/mod.rs index 9935a76..b3ddcd4 100644 --- a/net-guardia/src/ml/mod.rs +++ b/net-guardia/src/detection/ml/mod.rs @@ -5,4 +5,4 @@ pub mod feature_extractor; pub mod inference; pub mod engine; pub mod aggregator; -pub mod traffic_logger; \ No newline at end of file +pub mod traffic_logger; diff --git a/net-guardia/src/detection/ml/model_loader.rs b/net-guardia/src/detection/ml/model_loader.rs new file mode 100644 index 0000000..4625a1f --- /dev/null +++ b/net-guardia/src/detection/ml/model_loader.rs @@ -0,0 +1,58 @@ +use tract_onnx::prelude::*; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::core::infrastructure::app_config::AppConfig; +use crate::model::error::ml::MLError; +use crate::model::ml_detection::RunnableModel; + +use super::config_loader::InferenceConfig; + +pub struct MLModels { + pub deep_autoencoder: RunnableModel, +} + +impl MLModels { + pub fn load_models( + app_config: &Arc, + inference_config: &Arc, + ) -> Result { + Ok(Self { + deep_autoencoder: Self::load_lstm_ae( + &app_config.deep_autoencoder_name, + inference_config.window_size, + inference_config.num_ae_features(), + )?, + }) + } + + fn load_lstm_ae(model_name: &str, window_size: usize, features: usize) -> Result { + let model_path = PathBuf::from(env!("ARTIFACTCS_PATH")).join(model_name); + + let mut model = onnx() + .model_for_path(&model_path) + .map_err(|_| MLError::ModelLoadFailed { path: model_path.clone() })?; + + // LSTM AE: input shape = (batch=1, seq_len=window_size, features) + model + .set_input_fact(0, f32::fact(&[1, window_size, features]).into()) + .map_err(|_| MLError::ModelLoadFailed { path: model_path.clone() })?; + + model + .into_optimized() + .map_err(|_| MLError::ModelLoadFailed { path: model_path.clone() })? + .into_runnable() + .map_err(|_| MLError::ModelLoadFailed { path: model_path }) + } + + pub fn get_model_info(&self, name: &str) -> String { + match name { + "deep_autoencoder" => { + let inputs = self.deep_autoencoder.model().inputs.len(); + let outputs = self.deep_autoencoder.model().outputs.len(); + format!("{}: inputs: {}, outputs: {}", name, inputs, outputs) + } + _ => "unknown model".to_string(), + } + } +} \ No newline at end of file diff --git a/net-guardia/src/ml/traffic_logger.rs b/net-guardia/src/detection/ml/traffic_logger.rs similarity index 93% rename from net-guardia/src/ml/traffic_logger.rs rename to net-guardia/src/detection/ml/traffic_logger.rs index d027a22..cf7979a 100644 --- a/net-guardia/src/ml/traffic_logger.rs +++ b/net-guardia/src/detection/ml/traffic_logger.rs @@ -2,6 +2,8 @@ use std::fs::OpenOptions; use std::io::{BufWriter, Write}; use std::thread; +use tracing; + use crossbeam::channel::{bounded, Sender, TrySendError}; pub struct TrafficLogger { @@ -27,7 +29,7 @@ impl TrafficLogger { .spawn(move || { for record in receiver { if let Err(e) = writeln!(writer, "{}", record.join(",")) { - eprintln!("[traffic-logger] write error: {}", e); + tracing::warn!("[traffic-logger] write error: {}", e); } } let _ = writer.flush(); diff --git a/net-guardia/src/detection/mod.rs b/net-guardia/src/detection/mod.rs new file mode 100644 index 0000000..7fcb61d --- /dev/null +++ b/net-guardia/src/detection/mod.rs @@ -0,0 +1,2 @@ +pub mod ml; +pub mod rule; \ No newline at end of file diff --git a/net-guardia/src/detection/rule/app_layer/dns.rs b/net-guardia/src/detection/rule/app_layer/dns.rs new file mode 100644 index 0000000..aeac6b5 --- /dev/null +++ b/net-guardia/src/detection/rule/app_layer/dns.rs @@ -0,0 +1,101 @@ +/// Minimal DNS query parser — extracts the QNAME from the first question. +/// +/// DNS over UDP: the payload starts directly at the DNS message header. +/// DNS over TCP: the first two bytes are the message length prefix — we +/// skip them when the first two bytes look like a length (i.e. the payload +/// is longer than the length prefix value). + +#[derive(Debug, Default)] +pub struct DnsInfo { + /// First QNAME decoded to dotted-label ASCII. + pub query_name: Vec, +} + +pub fn parse_dns(data: &[u8]) -> Option { + // DNS header is 12 bytes. + if data.len() < 12 { + return None; + } + + // Heuristic: if this looks like DNS-over-TCP (length-prefixed), + // skip the 2-byte length prefix. + let base = if has_tcp_length_prefix(data) { 2 } else { 0 }; + let hdr = data.get(base..base + 12)?; + + // QR bit: must be 0 for a query (we also handle responses for dns_query buffer). + // QDCOUNT must be >= 1. + let qdcount = u16::from_be_bytes([hdr[4], hdr[5]]); + if qdcount == 0 { + return None; + } + + // Questions start after 12-byte header. + let mut pos = base + 12; + let name = decode_qname(data, &mut pos)?; + + Some(DnsInfo { query_name: name }) +} + +/// Decode a DNS QNAME starting at `*pos`, advancing it past the name and the +/// two-byte QTYPE/QCLASS fields. Returns the name in dotted-label form. +fn decode_qname(data: &[u8], pos: &mut usize) -> Option> { + let mut out: Vec = Vec::new(); + let mut jumps = 0usize; + let mut cur = *pos; + + loop { + if cur >= data.len() { + return None; + } + let len = data[cur] as usize; + + if len == 0 { + // End of name + cur += 1; + break; + } + + // Pointer compression: top 2 bits are 11. + if len & 0xC0 == 0xC0 { + if cur + 1 >= data.len() { return None; } + let offset = ((len & 0x3F) << 8 | data[cur + 1] as usize) as usize; + // Only update the caller's position on the first pointer. + if jumps == 0 { + *pos = cur + 2; + } + cur = offset; + jumps += 1; + if jumps > 10 { return None; } // avoid infinite loops + continue; + } + + cur += 1; + if cur + len > data.len() { return None; } + + if !out.is_empty() { + out.push(b'.'); + } + out.extend_from_slice(&data[cur..cur + len]); + cur += len; + } + + // Update caller's position only if no pointer compression was used. + if jumps == 0 { + *pos = cur; + } + + // Skip QTYPE (2) + QCLASS (2). + // (pos is now pointing past the name's null terminator) + // We don't update *pos further since callers only need the name. + + Some(out) +} + +/// True if the first two bytes look like a TCP DNS length prefix +/// (i.e. `data.len() >= u16_be(data[0..2]) + 2`). +fn has_tcp_length_prefix(data: &[u8]) -> bool { + if data.len() < 14 { return false; } + let len = u16::from_be_bytes([data[0], data[1]]) as usize; + // Plausible if the declared length fits in the buffer and is at least 12. + len >= 12 && len + 2 <= data.len() +} diff --git a/net-guardia/src/detection/rule/app_layer/http.rs b/net-guardia/src/detection/rule/app_layer/http.rs new file mode 100644 index 0000000..131b87b --- /dev/null +++ b/net-guardia/src/detection/rule/app_layer/http.rs @@ -0,0 +1,175 @@ +/// Lightweight HTTP/1.x request and response parser. +/// +/// Extracts the fields used by Suricata/Snort HTTP sticky buffers so that +/// content rules targeted at `http.uri`, `http.user_agent`, etc. can be +/// verified against the correct byte slice instead of the raw TCP payload. + +#[derive(Debug, Default)] +pub struct HttpRequest { + pub method: Vec, + pub uri: Vec, + pub raw_uri: Vec, + pub host: Vec, + pub user_agent: Vec, + pub cookie: Vec, + /// Full request header section (everything before the blank line). + pub headers: Vec, + /// Request body (everything after the blank line). + pub client_body: Vec, +} + +#[derive(Debug, Default)] +pub struct HttpResponse { + pub stat_code: Vec, + /// Full response header section. + pub headers: Vec, + /// Response body. + pub server_body: Vec, +} + +static HTTP_METHODS: &[&[u8]] = &[ + b"GET", b"POST", b"PUT", b"DELETE", b"HEAD", + b"OPTIONS", b"PATCH", b"CONNECT", b"TRACE", +]; + +/// Try to parse `data` as an HTTP/1.x request. +/// Returns `None` when the data does not start with a recognised method. +pub fn parse_http_request(data: &[u8]) -> Option { + // Find first space — end of method token. + let sp1 = data.iter().position(|&b| b == b' ')?; + let method = &data[..sp1]; + + // Validate method + if !HTTP_METHODS.contains(&method) { + return None; + } + + // Find second space — end of URI. + let after_sp1 = sp1 + 1; + let sp2 = data[after_sp1..].iter().position(|&b| b == b' ')?; + let uri = data[after_sp1..after_sp1 + sp2].to_vec(); + + // Find end of first line. + let crlf1 = find_crlf(data, after_sp1 + sp2)?; + let headers_start = crlf1 + 2; + + // Find end of headers (blank line). + let (headers_end, body_start) = find_header_end(data, headers_start); + let headers_bytes = &data[..headers_end]; // includes request line + + let mut req = HttpRequest { + method: method.to_vec(), + raw_uri: uri.clone(), + uri: decode_uri(&uri), + headers: headers_bytes.to_vec(), + client_body: data.get(body_start..).unwrap_or(&[]).to_vec(), + ..Default::default() + }; + + // Walk individual header lines to extract named fields. + let mut pos = headers_start; + while pos < headers_end { + let line_end = find_crlf(data, pos).unwrap_or(headers_end); + let line = &data[pos..line_end]; + extract_header_field(line, &mut req); + pos = line_end + 2; + if pos >= headers_end { break; } + } + + Some(req) +} + +/// Try to parse `data` as an HTTP/1.x response. +/// Returns `None` when the data does not start with `HTTP/`. +pub fn parse_http_response(data: &[u8]) -> Option { + if !data.starts_with(b"HTTP/") { + return None; + } + + // "HTTP/1.x NNN " — status code starts after version + space. + let sp1 = data.iter().position(|&b| b == b' ')?; + let code_start = sp1 + 1; + let sp2 = data[code_start..].iter().position(|&b| b == b' ' || b == b'\r' || b == b'\n') + .unwrap_or(3); // 3-digit code + let stat_code = data[code_start..code_start + sp2].to_vec(); + + let crlf1 = find_crlf(data, code_start + sp2)?; + let headers_start = crlf1 + 2; + let (headers_end, body_start) = find_header_end(data, headers_start); + + Some(HttpResponse { + stat_code, + headers: data[..headers_end].to_vec(), + server_body: data.get(body_start..).unwrap_or(&[]).to_vec(), + }) +} + +/// Populate named fields from a single HTTP header line (`Name: value`). +fn extract_header_field(line: &[u8], req: &mut HttpRequest) { + let colon = match line.iter().position(|&b| b == b':') { + Some(p) => p, + None => return, + }; + let name = &line[..colon]; + let value = line.get(colon + 1..).map(|v| v.trim_ascii_start()).unwrap_or(&[]); + + if name.eq_ignore_ascii_case(b"Host") { + req.host = value.to_vec(); + } else if name.eq_ignore_ascii_case(b"User-Agent") { + req.user_agent = value.to_vec(); + } else if name.eq_ignore_ascii_case(b"Cookie") { + req.cookie = value.to_vec(); + } +} + +/// Find the first CRLF (`\r\n`) in `data` starting at `from`. +/// Returns the offset of `\r`. +fn find_crlf(data: &[u8], from: usize) -> Option { + let slice = data.get(from..)?; + let pos = slice.windows(2).position(|w| w == b"\r\n")?; + Some(from + pos) +} + +/// Find end-of-headers (blank line `\r\n\r\n`) starting at `from`. +/// Returns `(headers_end, body_start)` where `headers_end` is the offset of +/// the first `\r` of the blank line, and `body_start` is after the blank line. +fn find_header_end(data: &[u8], from: usize) -> (usize, usize) { + let slice = match data.get(from..) { + Some(s) => s, + None => return (data.len(), data.len()), + }; + if let Some(pos) = slice.windows(4).position(|w| w == b"\r\n\r\n") { + let abs = from + pos; + (abs + 2, abs + 4) // end at blank line start; body after blank line + } else { + (data.len(), data.len()) + } +} + +/// Percent-decode a URI byte slice. +fn decode_uri(uri: &[u8]) -> Vec { + let mut out = Vec::with_capacity(uri.len()); + let mut i = 0; + while i < uri.len() { + if uri[i] == b'%' && i + 2 < uri.len() { + if let (Some(h), Some(l)) = (hex_val(uri[i + 1]), hex_val(uri[i + 2])) { + out.push((h << 4) | l); + i += 3; + continue; + } + } + out.push(uri[i]); + i += 1; + } + out +} + +#[inline] +fn hex_val(b: u8) -> Option { + match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + } +} diff --git a/net-guardia/src/detection/rule/app_layer/mod.rs b/net-guardia/src/detection/rule/app_layer/mod.rs new file mode 100644 index 0000000..c4e0fa1 --- /dev/null +++ b/net-guardia/src/detection/rule/app_layer/mod.rs @@ -0,0 +1,121 @@ +pub mod dns; +pub mod http; +pub mod tls; + +use http::{HttpRequest, HttpResponse}; + +pub use crate::model::rule_detection::{ParsedPacket, buf}; + +/// Well-known TLS server ports. When either sport or dport matches, the flow +/// is treated as TLS unconditionally — even for mid-stream captures where the +/// TCP payload no longer starts at a TLS record boundary. +fn is_tls_port(port: u16) -> bool { + matches!(port, + 443 // HTTPS + | 8443 // HTTPS alt + | 465 // SMTPS + | 993 // IMAPS + | 995 // POP3S + | 636 // LDAPS + | 853 // DNS-over-TLS + | 4433 // HTTPS alt + | 8883 // MQTT over TLS + ) +} + +/// Parse `data` into a `ParsedPacket`. +/// +/// Both `sport` and `dport` are needed so that we can identify TLS flows by +/// server port even when the callback only has the ephemeral client port stored +/// in `current_dport` (e.g. the flow key was created from an inbound Google +/// packet: sip=142.251.x.x:443 → dip=local:52873, so sport=443). +/// +/// Detection order: +/// 1. TLS — content-based header check OR port-based hint (catches mid-stream) +/// 2. HTTP request/response +/// 3. DNS (port 53 / 5353) +pub fn detect_and_parse(data: &[u8], sport: u16, dport: u16) -> ParsedPacket { + let raw = data.to_vec(); + let mut pkt = ParsedPacket { + raw, + tls_app_data: false, + is_tls: false, + http_uri: Vec::new(), + http_raw_uri: Vec::new(), + http_method: Vec::new(), + http_host: Vec::new(), + http_user_agent: Vec::new(), + http_cookie: Vec::new(), + http_header: Vec::new(), + http_client_body: Vec::new(), + http_stat_code: Vec::new(), + http_server_body: Vec::new(), + tls_sni: Vec::new(), + dns_query: Vec::new(), + }; + + if data.is_empty() { + return pkt; + } + + // 1. TLS — content-based detection first (handles normal cases, extracts SNI). + if let Some(tls) = tls::detect_tls(data) { + pkt.raw.clear(); + pkt.is_tls = true; + if tls.is_app_data { + pkt.tls_app_data = true; + return pkt; + } + pkt.tls_sni = tls.sni; + return pkt; + } + + // 2. HTTP request / response — must come BEFORE port-based TLS fallback + // so that plaintext HTTP on port 443 is identified correctly. + if let Some(req) = http::parse_http_request(data) { + apply_request(&mut pkt, req); + return pkt; + } + if let Some(resp) = http::parse_http_response(data) { + apply_response(&mut pkt, resp); + return pkt; + } + + // 3. DNS + if dport == 53 || dport == 5353 || sport == 53 || sport == 5353 { + if let Some(dns) = dns::parse_dns(data) { + pkt.dns_query = dns.query_name; + } + return pkt; + } + + // 4. Port-based TLS fallback — reached only when content detection found + // nothing. Handles mid-stream captures of TLS flows where the first + // byte is not a TLS record header (connection established before we + // started capturing). Suricata equivalent: AppLayerProtoDetectGetProto + // with port hints as last resort. + if is_tls_port(sport) || is_tls_port(dport) { + pkt.raw.clear(); + pkt.is_tls = true; + return pkt; + } + + pkt +} + +fn apply_request(pkt: &mut ParsedPacket, req: HttpRequest) { + pkt.http_uri = req.uri; + pkt.http_raw_uri = req.raw_uri; + pkt.http_method = req.method; + pkt.http_host = req.host; + pkt.http_user_agent = req.user_agent; + pkt.http_cookie = req.cookie; + pkt.http_header = req.headers; + pkt.http_client_body = req.client_body; +} + +fn apply_response(pkt: &mut ParsedPacket, resp: HttpResponse) { + pkt.http_stat_code = resp.stat_code; + pkt.http_header = resp.headers; + pkt.http_server_body = resp.server_body; +} diff --git a/net-guardia/src/detection/rule/app_layer/tls.rs b/net-guardia/src/detection/rule/app_layer/tls.rs new file mode 100644 index 0000000..f5794d8 --- /dev/null +++ b/net-guardia/src/detection/rule/app_layer/tls.rs @@ -0,0 +1,141 @@ +/// TLS record-layer parser. +/// +/// Detects TLS by the 5-byte record header and extracts the SNI hostname +/// from a ClientHello handshake message. Most importantly, it identifies +/// TLS Application Data records (type 0x17) so that the rule engine can +/// skip scanning encrypted payload — the primary source of false positives. + +/// Recognised TLS record content types. +const RT_CHANGE_CIPHER: u8 = 0x14; +const RT_ALERT: u8 = 0x15; +const RT_HANDSHAKE: u8 = 0x16; +const RT_APP_DATA: u8 = 0x17; + +const HS_CLIENT_HELLO: u8 = 0x01; +const EXT_SNI: u16 = 0x0000; + +#[derive(Debug, Default)] +pub struct TlsInfo { + /// True when the first record in `data` is Application Data (encrypted). + /// The rule engine MUST skip all content scanning for such payloads. + pub is_app_data: bool, + /// SNI hostname bytes extracted from a ClientHello, if present. + pub sni: Vec, +} + +/// Try to parse `data` as a TLS record stream. +/// Returns `None` when `data` does not look like a TLS record. +pub fn detect_tls(data: &[u8]) -> Option { + if data.len() < 5 { + return None; + } + + let record_type = data[0]; + if !matches!(record_type, RT_CHANGE_CIPHER | RT_ALERT | RT_HANDSHAKE | RT_APP_DATA) { + return None; + } + + // TLS major version must be 3. + if data[1] != 3 { + return None; + } + // Minor version 0-4 (SSL3..TLS1.3). + if data[2] > 4 { + return None; + } + + if record_type == RT_APP_DATA { + return Some(TlsInfo { is_app_data: true, sni: Vec::new() }); + } + + let mut info = TlsInfo::default(); + + if record_type == RT_HANDSHAKE { + info.sni = extract_sni(data); + } + + Some(info) +} + +/// Extract the SNI hostname from a TLS ClientHello record. +/// Returns an empty Vec when SNI is absent or the record is malformed. +fn extract_sni(data: &[u8]) -> Vec { + // TLS record header: type(1) + version(2) + length(2) = 5 bytes + // Handshake header: type(1) + length(3) = 4 bytes + // ClientHello: version(2) + random(32) + session_id_len(1) + ... + if data.len() < 5 + 4 + 2 + 32 + 1 { + return Vec::new(); + } + + let hs = &data[5..]; // start of Handshake layer + if hs[0] != HS_CLIENT_HELLO { + return Vec::new(); + } + + let hs_len = u24_be(&hs[1..4]) as usize; + if hs.len() < 4 + hs_len { + return Vec::new(); + } + + let ch = &hs[4..4 + hs_len]; // ClientHello body + // version(2) + random(32) = 34 bytes minimum + if ch.len() < 34 { + return Vec::new(); + } + + let mut pos = 34usize; // skip version + random + + // Session ID + if pos >= ch.len() { return Vec::new(); } + let sid_len = ch[pos] as usize; + pos += 1 + sid_len; + + // Cipher suites + if pos + 2 > ch.len() { return Vec::new(); } + let cs_len = u16_be(&ch[pos..]) as usize; + pos += 2 + cs_len; + + // Compression methods + if pos >= ch.len() { return Vec::new(); } + let cm_len = ch[pos] as usize; + pos += 1 + cm_len; + + // Extensions + if pos + 2 > ch.len() { return Vec::new(); } + let ext_total = u16_be(&ch[pos..]) as usize; + pos += 2; + let ext_end = pos + ext_total; + if ext_end > ch.len() { return Vec::new(); } + + while pos + 4 <= ext_end { + let ext_type = u16_be(&ch[pos..]); + let ext_len = u16_be(&ch[pos + 2..]) as usize; + pos += 4; + if pos + ext_len > ext_end { break; } + + if ext_type == EXT_SNI && ext_len >= 5 { + // SNI list: list_len(2) + name_type(1) + name_len(2) + name + let list_len = u16_be(&ch[pos..]) as usize; + if list_len + 2 > ext_len { break; } + let name_type = ch[pos + 2]; + let name_len = u16_be(&ch[pos + 3..]) as usize; + if name_type == 0 && pos + 5 + name_len <= ext_end { + return ch[pos + 5..pos + 5 + name_len].to_vec(); + } + } + + pos += ext_len; + } + + Vec::new() +} + +#[inline] +fn u16_be(b: &[u8]) -> u16 { + u16::from_be_bytes([b[0], b[1]]) +} + +#[inline] +fn u24_be(b: &[u8]) -> u32 { + (b[0] as u32) << 16 | (b[1] as u32) << 8 | b[2] as u32 +} diff --git a/net-guardia/src/detection/rule/mod.rs b/net-guardia/src/detection/rule/mod.rs new file mode 100644 index 0000000..a4319b0 --- /dev/null +++ b/net-guardia/src/detection/rule/mod.rs @@ -0,0 +1,3 @@ +pub mod app_layer; +pub mod rule_engine; +pub mod stream_reassembler; \ No newline at end of file diff --git a/net-guardia/src/detection/rule/rule_engine.rs b/net-guardia/src/detection/rule/rule_engine.rs new file mode 100644 index 0000000..07c182b --- /dev/null +++ b/net-guardia/src/detection/rule/rule_engine.rs @@ -0,0 +1,315 @@ +use std::path::PathBuf; +use rusqlite::{Connection, params}; +use vectorscan_rs::{BlockDatabase, Flag, Pattern, Scan}; + +use crate::model::error::rule::RuleError; +use crate::model::rule_detection::{RuleMatch, RuleSig, ContentEntry, ParsedPacket}; + +pub struct RuleEngine { + db: BlockDatabase, + sigs: Vec, +} + +impl RuleEngine { + pub fn new() -> Option { + let conn = Connection::open(PathBuf::from(env!("RULES_DB_PATH")).join("rules.db")).ok()?; + let patterns = load_patterns(&conn)?; + let sigs = load_sigs(&conn)?; + + if patterns.is_empty() { + return None; + } + + let db = match BlockDatabase::new(patterns) { + Ok(db) => db, + Err(e) => { + RuleError::BlockDatabaseFailed { error: e.to_string() }; + return None; + } + }; + + Some(Self { db, sigs }) + } + + pub fn pattern_count(&self) -> usize { + self.sigs.len() + } + + /// Scan a parsed application-layer packet. + /// + /// Returns an empty list immediately for TLS Application Data (encrypted) + /// because scanning random cipher bytes produces only false positives. + pub fn scan(&self, pkt: &ParsedPacket, is_ingress: bool, dst_port: u16) -> Vec { + // Never scan encrypted TLS application data. + if pkt.tls_app_data { + return Vec::new(); + } + + if pkt.raw.is_empty() { + return Vec::new(); + } + + let mut scanner = match self.db.create_scanner() { + Ok(s) => s, + Err(_) => return Vec::new(), + }; + + // Prefilter: run vectorscan against the raw payload. + // For HTTP traffic the raw bytes include the full request/response, so + // URI / header patterns are still found here. Buffer-specific chain + // verification then confirms using the correct parsed field. + let mut candidates: Vec = Vec::new(); + let _ = scanner.scan(&pkt.raw, |id, _from, _to, _flags| { + candidates.push(id); + Scan::Continue + }); + candidates.sort_unstable(); + candidates.dedup(); + + let mut results = Vec::new(); + + for sig_idx in candidates { + let Some(sig) = self.sigs.get(sig_idx as usize) else { + continue; + }; + + // Direction filter + let dir_ok = match sig.flow_dir { + 1 => is_ingress, + 2 => !is_ingress, + _ => true, + }; + if !dir_ok { + continue; + } + + // Port filter + if !sig.dports.is_empty() && !sig.dports.contains(&dst_port) { + continue; + } + + // Full chain verification with per-entry buffer routing. + if verify_chain(&sig.chain, pkt) { + results.push(RuleMatch { + sid: sig.sid, + msg: sig.msg.clone(), + src: String::new(), + dst: String::new(), + }); + } + } + + results + } +} + +// ─── SQLite loaders ─────────────────────────────────────────────────────────── + +fn load_patterns(conn: &Connection) -> Option> { + let mut stmt = conn + .prepare("SELECT id, expression, nocase FROM patterns ORDER BY id") + .ok()?; + let patterns: Vec = stmt + .query_map([], |row| { + let id: i64 = row.get(0)?; + let expression: Vec = row.get(1)?; + let nocase: i64 = row.get(2)?; + let flags = if nocase != 0 { Flag::CASELESS } else { Flag::default() }; + Ok(Pattern::new(expression, flags, Some(id as u32))) + }) + .ok()? + .filter_map(|r| r.ok()) + .collect(); + Some(patterns) +} + +fn load_sigs(conn: &Connection) -> Option> { + let mut stmt = conn + .prepare("SELECT id, sid, flow_dir, msg FROM signatures ORDER BY id") + .ok()?; + let sig_rows: Vec<(i64, u32, u8, String)> = stmt + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, u32>(1)?, + row.get::<_, u8>(2)?, + row.get::<_, String>(3)?, + )) + }) + .ok()? + .filter_map(|r| r.ok()) + .collect(); + + let mut sigs = Vec::with_capacity(sig_rows.len()); + for (sig_id, sid, flow_dir, msg) in sig_rows { + let mut port_stmt = conn + .prepare("SELECT dport FROM signature_ports WHERE sig_id = ?1") + .ok()?; + let dports: Vec = port_stmt + .query_map(params![sig_id], |row| row.get::<_, u16>(0)) + .ok()? + .filter_map(|r| r.ok()) + .collect(); + + let mut entry_stmt = conn + .prepare( + "SELECT pattern, nocase, negated, \ + has_distance, has_within, has_offset, has_depth, \ + distance, within, off, depth, buffer \ + FROM content_entries WHERE sig_id = ?1 ORDER BY entry_order", + ) + .ok()?; + let chain: Vec = entry_stmt + .query_map(params![sig_id], |row| { + Ok(ContentEntry { + pattern: row.get(0)?, + nocase: row.get::<_, i64>(1)? != 0, + negated: row.get::<_, i64>(2)? != 0, + has_distance: row.get::<_, i64>(3)? != 0, + has_within: row.get::<_, i64>(4)? != 0, + has_offset: row.get::<_, i64>(5)? != 0, + has_depth: row.get::<_, i64>(6)? != 0, + distance: row.get::<_, i32>(7)?, + within: row.get::<_, i32>(8)?, + offset: row.get::<_, u16>(9)?, + depth: row.get::<_, u16>(10)?, + buffer: row.get::<_, u8>(11)?, + }) + }) + .ok()? + .filter_map(|r| r.ok()) + .collect(); + + sigs.push(RuleSig { sid, flow_dir, dports, msg, chain }); + } + + Some(sigs) +} + +// ─── Chain verification ─────────────────────────────────────────────────────── + +fn verify_chain(chain: &[ContentEntry], pkt: &ParsedPacket) -> bool { + verify_from(chain, pkt, 0, 0) +} + +/// Recursively verify the content chain starting at `idx` with `prev_end` as +/// the anchor for relative constraints. +/// +/// Each `ContentEntry` carries a `buffer` id; entries within the same chain +/// can target different buffers. When a required buffer is empty (e.g. no +/// HTTP was detected), a non-negated entry immediately fails the chain. +fn verify_from(chain: &[ContentEntry], pkt: &ParsedPacket, idx: usize, prev_end: usize) -> bool { + if idx >= chain.len() { + return true; + } + + let entry = &chain[idx]; + let payload = pkt.buffer(entry.buffer); + + // If the required buffer was not populated (e.g. http_uri for non-HTTP + // traffic), the pattern can never match → fail any non-negated chain. + if payload.is_empty() && !entry.negated { + return false; + } + + let buf_len = payload.len(); + let (ws, we) = compute_window(entry, prev_end, buf_len); + + // Degenerate window + if ws >= we || ws >= buf_len { + return if entry.negated { + verify_from(chain, pkt, idx + 1, prev_end) + } else { + false + }; + } + + let window = &payload[ws..we]; + + if entry.negated { + if find_in(window, &entry.pattern, entry.nocase).is_some() { + return false; + } + return verify_from(chain, pkt, idx + 1, prev_end); + } + + let next_is_relative = idx + 1 < chain.len() && { + let next = &chain[idx + 1]; + next.has_distance || next.has_within + }; + + let mut search_from = 0usize; + loop { + let sub = &window[search_from..]; + let Some(rel_pos) = find_in(sub, &entry.pattern, entry.nocase) else { + break; + }; + + let match_end = ws + search_from + rel_pos + entry.pattern.len(); + + if verify_from(chain, pkt, idx + 1, match_end) { + return true; + } + + if !next_is_relative { + break; + } + + search_from += rel_pos + 1; + if search_from >= window.len() { + break; + } + } + + false +} + +/// Compute the [start, end) search window for a content entry. +fn compute_window(entry: &ContentEntry, prev_end: usize, buf_len: usize) -> (usize, usize) { + let (mut start, mut end) = if entry.has_distance || entry.has_within { + let dist = entry.distance; + let s = if dist >= 0 { + prev_end.saturating_add(dist as usize) + } else { + prev_end.saturating_sub((-dist) as usize) + }; + let e = if entry.has_within { + let dist_non_neg = dist.max(0) as usize; + prev_end.saturating_add(dist_non_neg).saturating_add(entry.within as usize) + } else { + buf_len + }; + (s, e) + } else { + let s = if entry.has_offset { entry.offset as usize } else { 0 }; + let e = if entry.has_depth { entry.depth as usize } else { buf_len }; + (s, e) + }; + + if entry.has_offset { + start = start.max(entry.offset as usize); + } + if entry.has_depth { + end = end.min(entry.depth as usize); + } + + (start, end.min(buf_len)) +} + +fn find_in(haystack: &[u8], needle: &[u8], nocase: bool) -> Option { + if needle.is_empty() { + return Some(0); + } + if haystack.len() < needle.len() { + return None; + } + + if nocase { + let needle_lc: Vec = needle.iter().map(|b| b.to_ascii_lowercase()).collect(); + haystack + .windows(needle.len()) + .position(|w| w.iter().zip(&needle_lc).all(|(a, b)| a.to_ascii_lowercase() == *b)) + } else { + haystack.windows(needle.len()).position(|w| w == needle) + } +} \ No newline at end of file diff --git a/net-guardia/src/detection/rule/stream_reassembler.rs b/net-guardia/src/detection/rule/stream_reassembler.rs new file mode 100644 index 0000000..dc13606 --- /dev/null +++ b/net-guardia/src/detection/rule/stream_reassembler.rs @@ -0,0 +1,389 @@ +use std::cell::RefCell; +use std::collections::HashMap; +use std::ffi::c_void; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::rc::Rc; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use protolens::{Prolens, TransProto}; + +use macros::log; + +use super::app_layer; +use super::rule_engine::RuleEngine; +use crate::model::error::rule::RuleError; +use crate::model::rule_detection::{FlowKey, FlowProto, FlowState, RawPkt, RuleMatch}; + + +/// Per-thread TCP stream reassembler backed by protolens. +/// Not Send (Prolens uses Rc/RefCell) — must be created inside the XSK thread. +pub struct StreamReassembler { + engine: Prolens, + flows: HashMap, + rule_engine: Arc, + pending: Rc>>, + + /// Source port of the active flow — set before each run_task call. + current_sport: Rc>, + + /// Destination port of the active flow. + current_dport: Rc>, + + /// Protocol already identified for the active flow (FlowProto as u8). + /// Callbacks write back here when they first detect the protocol so that + /// process_tcp() can persist it into FlowState.detected_proto. + current_proto: Rc>, + + /// (flow_key, sid) -> cumulative match count for min_matches threshold + flow_sid_counts: HashMap<(FlowKey, u32), u32>, + min_matches: u32, + is_ingress: bool, +} + +impl StreamReassembler { + pub fn new(rule_engine: Arc, min_matches: u32, is_ingress: bool) -> Self { + let pending = Rc::new(RefCell::new(Vec::::new())); + let current_sport = Rc::new(RefCell::new(0u16)); + let current_dport = Rc::new(RefCell::new(0u16)); + let current_proto = Rc::new(RefCell::new(0u8)); + + let p_c2s = Rc::clone(&pending); + let re_c2s = Arc::clone(&rule_engine); + let sp_c2s = Rc::clone(¤t_sport); + let port_c2s = Rc::clone(¤t_dport); + let pr_c2s = Rc::clone(¤t_proto); + + let p_s2c = Rc::clone(&pending); + let re_s2c = Arc::clone(&rule_engine); + let sp_s2c = Rc::clone(¤t_sport); + let port_s2c = Rc::clone(¤t_dport); + let pr_s2c = Rc::clone(¤t_proto); + + let mut engine = Prolens::::default(); + + engine.set_cb_task_c2s(move |data: &[u8], _seq: u32, _ctx: *const c_void| { + // If the flow was already identified as TLS, skip entirely — + // mirrors Suricata's per-flow alproto check. + if *pr_c2s.borrow() == FlowProto::Tls as u8 { + return; + } + let sport = *sp_c2s.borrow(); + let dport = *port_c2s.borrow(); + let pkt = app_layer::detect_and_parse(data, sport, dport); + if pkt.is_tls { + *pr_c2s.borrow_mut() = FlowProto::Tls as u8; + return; + } + p_c2s.borrow_mut().extend(re_c2s.scan(&pkt, true, dport)); + }); + + engine.set_cb_task_s2c(move |data: &[u8], _seq: u32, _ctx: *const c_void| { + if *pr_s2c.borrow() == FlowProto::Tls as u8 { + return; + } + let sport = *sp_s2c.borrow(); + let dport = *port_s2c.borrow(); + let pkt = app_layer::detect_and_parse(data, sport, dport); + if pkt.is_tls { + *pr_s2c.borrow_mut() = FlowProto::Tls as u8; + return; + } + p_s2c.borrow_mut().extend(re_s2c.scan(&pkt, false, dport)); + }); + + Self { + engine, + flows: HashMap::new(), + rule_engine, + pending, + current_sport, + current_dport, + current_proto, + flow_sid_counts: HashMap::new(), + min_matches, + is_ingress, + } + } + + /// Feed a raw Ethernet frame. Returns alerts that have crossed min_matches threshold. + pub fn process(&mut self, packet_data: &[u8]) -> Result, RuleError> { + let Some(pkt) = parse_raw(packet_data) else { + return Ok(Vec::new()); + }; + + // Skip multicast and broadcast destinations. + // C2 malware never communicates via multicast; scanning these produces + // only false positives (SSDP, mDNS, LLMNR, OSPF, PIM, etc.). + if is_multicast_or_broadcast(&pkt.dip) { + return Ok(Vec::new()); + } + + let src = format!("{}:{}", pkt.sip, pkt.sport); + let dst = format!("{}:{}", pkt.dip, pkt.dport); + let flow_key = FlowKey { + sip: pkt.sip, + sport: pkt.sport, + dip: pkt.dip, + dport: pkt.dport, + }; + + let raw_matches = if matches!(pkt.trans_proto, TransProto::Udp) { + let parsed = app_layer::detect_and_parse(&pkt.payload, pkt.sport, pkt.dport); + self.rule_engine.scan(&parsed, self.is_ingress, pkt.dport) + } else { + self.process_tcp(pkt, &flow_key)? + }; + + Ok(self.apply_threshold(raw_matches, &flow_key, src, dst)) + } + + fn process_tcp(&mut self, pkt: RawPkt, key: &FlowKey) -> Result, RuleError> { + let rev = key.reversed(); + let is_rst = pkt.rst; + let is_fin = pkt.fin; + + let actual_key = if self.flows.contains_key(key) { + key.clone() + } else if self.flows.contains_key(&rev) { + rev + } else { + let task = self.engine.new_task(TransProto::Tcp); + self.flows.insert( + key.clone(), + FlowState { + task, + last_seen: Instant::now(), + fin_count: 0, + rst_seen: false, + detected_proto: FlowProto::Unknown, + }, + ); + key.clone() + }; + + let Some(state) = self.flows.get_mut(&actual_key) else { + return Err(RuleError::FlowStateMissing); + }; + state.last_seen = Instant::now(); + if is_rst { + state.rst_seen = true; + } + if is_fin { + state.fin_count += 1; + } + + // Restore per-flow protocol state so callbacks can fast-path TLS flows. + *self.current_sport.borrow_mut() = actual_key.sport; + *self.current_dport.borrow_mut() = actual_key.dport; + *self.current_proto.borrow_mut() = state.detected_proto as u8; + + self.engine.run_task(&mut state.task, pkt); + + // Persist any protocol the callbacks just identified back into FlowState. + let proto_byte = *self.current_proto.borrow(); + if let Some(s) = self.flows.get_mut(&actual_key) { + if s.detected_proto == FlowProto::Unknown && proto_byte != 0 { + s.detected_proto = if proto_byte == FlowProto::Tls as u8 { + FlowProto::Tls + } else { + FlowProto::Unknown + }; + } + } + + let matches: Vec = self.pending.borrow_mut().drain(..).collect(); + + if self + .flows + .get(&actual_key) + .map(|s| s.rst_seen || s.fin_count >= 2) + .unwrap_or(false) + { + self.flows.remove(&actual_key); + } + + Ok(matches) + } + + fn apply_threshold(&mut self, raw: Vec, flow_key: &FlowKey, src: String, dst: String) -> Vec { + let threshold = self.min_matches.max(1); + let mut alerts = Vec::new(); + for mut m in raw { + let count = self.flow_sid_counts.entry((flow_key.clone(), m.sid)).or_insert(0); + *count += 1; + if *count == threshold { + m.src = src.clone(); + m.dst = dst.clone(); + alerts.push(m); + } + } + alerts + } + + /// Drop flows idle longer than `max_idle`. Call periodically from the XSK thread. + pub fn cleanup_stale(&mut self, max_idle: Duration) { + let now = Instant::now(); + self.flows.retain(|_, s| now.duration_since(s.last_seen) < max_idle); + self.flow_sid_counts.retain(|(k, _), _| self.flows.contains_key(k)); + } + + pub fn flow_count(&self) -> usize { + self.flows.len() + } + + /// Match counts per (flow, sid) — for periodic stats logging. + pub fn match_counts(&self) -> Vec { + self.flow_sid_counts.values().cloned().collect() + } + + pub fn min_matches(&self) -> u32 { + self.min_matches + } +} + +// Raw frame parser + +fn parse_raw(data: &[u8]) -> Option { + if data.len() < 14 { + return None; + } + match u16::from_be_bytes([data[12], data[13]]) { + 0x0800 => parse_ipv4(data), + 0x86DD => parse_ipv6(data), + _ => None, + } +} + +fn parse_ipv4(data: &[u8]) -> Option { + if data.len() < 34 { + return None; + } + let ip = &data[14..]; + let proto = ip[9]; + if proto != 6 && proto != 17 { + return None; + } + + let sip = IpAddr::V4(Ipv4Addr::from(u32::from_be_bytes([ip[12], ip[13], ip[14], ip[15]]))); + let dip = IpAddr::V4(Ipv4Addr::from(u32::from_be_bytes([ip[16], ip[17], ip[18], ip[19]]))); + let ihl = (ip[0] & 0x0F) as usize * 4; + let total_len = u16::from_be_bytes([ip[2], ip[3]]) as usize; + + if data.len() < 14 + ihl + 4 { + return None; + } + let tp = &ip[ihl..]; + let sport = u16::from_be_bytes([tp[0], tp[1]]); + let dport = u16::from_be_bytes([tp[2], tp[3]]); + + if proto == 6 { + if data.len() < 14 + ihl + 20 { + return None; + } + let seq = u32::from_be_bytes([tp[4], tp[5], tp[6], tp[7]]); + let hlen = (tp[12] >> 4) as usize * 4; + let flags = tp[13]; + let payload_start = 14 + ihl + hlen; + let payload_end = (14 + total_len).min(data.len()); + Some(RawPkt { + trans_proto: TransProto::Tcp, + sip, + dip, + sport, + dport, + seq, + syn: flags & 0x02 != 0, + fin: flags & 0x01 != 0, + rst: flags & 0x04 != 0, + payload: data.get(payload_start..payload_end).unwrap_or(&[]).to_vec(), + }) + } else { + let payload_start = 14 + ihl + 8; + let payload_end = (14 + total_len).min(data.len()); + Some(RawPkt { + trans_proto: TransProto::Udp, + sip, + dip, + sport, + dport, + seq: 0, + syn: false, + fin: false, + rst: false, + payload: data.get(payload_start..payload_end).unwrap_or(&[]).to_vec(), + }) + } +} + +fn parse_ipv6(data: &[u8]) -> Option { + if data.len() < 54 { + return None; + } + let ip = &data[14..]; + let proto = ip[6]; + if proto != 6 && proto != 17 { + return None; + } + + let mut s = [0u8; 16]; + s.copy_from_slice(&ip[8..24]); + let mut d = [0u8; 16]; + d.copy_from_slice(&ip[24..40]); + let sip = IpAddr::V6(Ipv6Addr::from(s)); + let dip = IpAddr::V6(Ipv6Addr::from(d)); + + let payload_len = u16::from_be_bytes([ip[4], ip[5]]) as usize; + let tp = &ip[40..]; + let sport = u16::from_be_bytes([tp[0], tp[1]]); + let dport = u16::from_be_bytes([tp[2], tp[3]]); + + if proto == 6 { + if data.len() < 54 + 20 { + return None; + } + let seq = u32::from_be_bytes([tp[4], tp[5], tp[6], tp[7]]); + let hlen = (tp[12] >> 4) as usize * 4; + let flags = tp[13]; + let payload_start = 14 + 40 + hlen; + let payload_end = (14 + 40 + payload_len).min(data.len()); + Some(RawPkt { + trans_proto: TransProto::Tcp, + sip, + dip, + sport, + dport, + seq, + syn: flags & 0x02 != 0, + fin: flags & 0x01 != 0, + rst: flags & 0x04 != 0, + payload: data.get(payload_start..payload_end).unwrap_or(&[]).to_vec(), + }) + } else { + let payload_start = 14 + 40 + 8; + let payload_end = (14 + 40 + payload_len).min(data.len()); + Some(RawPkt { + trans_proto: TransProto::Udp, + sip, + dip, + sport, + dport, + seq: 0, + syn: false, + fin: false, + rst: false, + payload: data.get(payload_start..payload_end).unwrap_or(&[]).to_vec(), + }) + } +} + +/// True for IPv4 multicast (224.0.0.0/4), limited broadcast (255.255.255.255), +/// and IPv6 multicast (ff00::/8). +/// These destinations are never used by unicast C2 malware; scanning them +/// generates only false positives from UPnP/SSDP, mDNS, LLMNR, OSPF, etc. +fn is_multicast_or_broadcast(ip: &IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => v4.is_multicast() || v4.is_broadcast(), + IpAddr::V6(v6) => v6.is_multicast(), + } +} diff --git a/net-guardia/src/main.rs b/net-guardia/src/main.rs index d44d694..392d848 100644 --- a/net-guardia/src/main.rs +++ b/net-guardia/src/main.rs @@ -2,7 +2,7 @@ mod core; mod model; mod utils; mod web; -mod ml; +mod detection; use crate::core::system::System; use crate::model::error::Error; diff --git a/net-guardia/src/ml/inference.rs b/net-guardia/src/ml/inference.rs deleted file mode 100644 index ecdf1a4..0000000 --- a/net-guardia/src/ml/inference.rs +++ /dev/null @@ -1,140 +0,0 @@ -use std::sync::{Arc, Mutex}; - -use macros::log; -use tract_onnx::prelude::*; - -use super::config_loader::InferenceConfig; -use super::feature_extractor::FlowFeatures; -use super::flow_tracker::FlowData; -use super::model_loader::MLModels; -use crate::model::log::ml::MLLog; -use crate::model::ml_detection::DetectionResult; - -pub struct Inference { - pub models: Arc, - pub config: Arc, -} - -impl Inference { - pub fn new(models: Arc, config: Arc) -> Self { - Self { models, config } - } - - pub fn infer_batch(&self, flows: &[FlowData]) -> Vec { - flows.iter().filter_map(|flow| self.infer_single(flow)).collect() - } - - pub fn infer_single(&self, flow: &FlowData) -> Option { - // extract - let ae_features = self.preprocess_ae_features(flow); - - // 2. Deep Autoencoder - let ae_input = Self::vec_to_array2(&ae_features); - let ae_score = match self.run_autoencoder(&ae_input) { - Ok(score) => score, - Err(e) => { - log!(MLLog::InferenceFailed("DeepAutoEncoder".to_string(), e.to_string())); - return None; - } - }; - - let cls_input = self.build_classifier_input(&ae_features, ae_score); - - let (attack_type, confidence) = match self.run_classifier(&cls_input) { - Ok(result) => result, - Err(e) => { - log!(MLLog::InferenceFailed("LightGBM".to_string(), e.to_string())); - return None; - } - }; - - let is_attack = ae_score >= self.config.ae_threshold; - - let flow_key = format!( - "{}:{} -> {}:{} (proto {}) [{}]", - flow.flow_key.src_ip, - flow.flow_key.src_port, - flow.flow_key.dst_ip, - flow.flow_key.dst_port, - flow.flow_key.protocol, - flow.direction - ); - - Some(DetectionResult { - flow_key, - flow_key_raw: flow.flow_key.clone(), - direction: flow.direction, - is_attack, - attack_type: if is_attack { Some(attack_type) } else { None }, - confidence, - ae_score, - threshold: self.config.ae_threshold, - }) - } - - fn preprocess_ae_features(&self, flow: &FlowData) -> Vec { - let mut features = FlowFeatures::extract(flow, &self.config.ae_feature_names); - features.winsorize(&self.config.ae_clip_params, &self.config.ae_feature_names); - features.normalize(&self.config.ae_scaler_mean, &self.config.ae_scaler_std); - features.clip(self.config.ae_post_clip_min, self.config.ae_post_clip_max); - features.features.iter().map(|&x| x as f32).collect() - } - - fn vec_to_array2(v: &[f32]) -> tract_ndarray::Array2 { - tract_ndarray::Array2::from_shape_fn((1, v.len()), |(_, j)| v[j]) - } - - /// Classifier 輸入 = 已預處理的 ae_features ++ [ae_anomaly_score] - fn build_classifier_input(&self, ae_features: &[f32], ae_score: f32) -> tract_ndarray::Array2 { - let n = ae_features.len() + 1; - tract_ndarray::Array2::from_shape_fn((1, n), |(_, j)| { - if j < ae_features.len() { - ae_features[j] - } else { - ae_score - } - }) - } - - fn run_autoencoder(&self, input: &tract_ndarray::Array2) -> TractResult { - let result = self - .models - .deep_autoencoder - .run(tvec![input.clone().into_tensor().into()])?; - - let output = result[0] - .to_array_view::()? - .into_dimensionality::()?; - - let diff = input - &output; - let mse = (&diff * &diff).sum() / self.config.ae_feature_names.len() as f32; - - Ok(mse) - } - - fn run_classifier(&self, input: &tract_ndarray::Array2) -> TractResult<(String, f32)> { - let input_tensor = input.clone().into_tensor(); - let result = self.models.classifier.run(tvec![input_tensor.into()])?; - - let output = result[0].to_array_view::()?; - - let mut max_prob: f32 = 0.0; - let mut predicted_class: usize = 0; - - for (i, &prob) in output.iter().enumerate() { - if prob > max_prob { - max_prob = prob; - predicted_class = i; - } - } - - let attack_type = self - .config - .attack_labels - .get(&predicted_class.to_string()) - .cloned() - .unwrap_or_else(|| "UNKNOWN".to_string()); - - Ok((attack_type, max_prob)) - } -} diff --git a/net-guardia/src/ml/model_loader.rs b/net-guardia/src/ml/model_loader.rs deleted file mode 100644 index 7761262..0000000 --- a/net-guardia/src/ml/model_loader.rs +++ /dev/null @@ -1,60 +0,0 @@ -use tract_onnx::prelude::*; -use std::path::PathBuf; - -use crate::core::infrastructure::app_config::AppConfig; -use crate::model::error::ml::MLError; -use crate::model::ml_detection::RunnableModel; - -use super::config_loader::InferenceConfig; - -pub struct MLModels { - pub deep_autoencoder: RunnableModel, - pub classifier: RunnableModel, -} -impl MLModels { - pub fn load_models(app_config: &Arc, inference_config: &Arc) -> Result { - Ok(Self { - deep_autoencoder: Self::loader(&app_config.deep_autoencoder_name, inference_config.num_ae_features())?, - classifier: Self::loader(&app_config.classifier_name, inference_config.num_classifier_features())? - }) - } - - pub fn loader(model: &str, features: usize) -> Result { - let model_path = PathBuf::from("models").join(model); - - let mut model = onnx() - .model_for_path(&model_path) - .map_err(|_| { - MLError::ModelLoadFailed { path: model_path.clone() } - })?; - - model.set_input_fact(0, f32::fact(&[1, features]).into()) - .map_err(|_| { - MLError::ModelLoadFailed { path: model_path.clone() } - })?; - - let runnable_model = model - .into_optimized() - .map_err(|_| { - MLError::ModelLoadFailed { path: model_path.clone() } - })? - .into_runnable() - .map_err(|_| { - MLError::ModelLoadFailed { path: model_path } - })?; - - Ok(runnable_model) - } - - pub fn get_model_info(&self, name: &str) -> String { - let model = match name { - "deep_autoencoder" => &self.deep_autoencoder, - "classifier" => &self.classifier, - _ => return "unknown model".to_string(), - }; - - let inputs = model.model().inputs.len(); - let outputs = model.model().outputs.len(); - format!("{}: inputs: {}, outputs: {}", name, inputs, outputs) - } -} \ No newline at end of file diff --git a/net-guardia/src/model/config.rs b/net-guardia/src/model/config.rs index 12223ed..36eb9df 100644 --- a/net-guardia/src/model/config.rs +++ b/net-guardia/src/model/config.rs @@ -12,7 +12,6 @@ pub struct Config { pub egress_ifname: String, pub geoip_db_name: String, pub deep_autoencoder_name: String, - pub classifier_name: String, pub models_config_name: String, pub combined_queue_count: u32, pub channel_size: usize, @@ -29,6 +28,8 @@ pub struct Config { pub inference_interval_secs: u64, pub aggregator_window_secs: u64, pub inference_batch_size: usize, + pub flow_timeout_us: u64, pub traffic_logging_mode: bool, pub traffic_log_csv_path: String, + pub min_signature_matches: u32, } \ No newline at end of file diff --git a/net-guardia/src/model/direction.rs b/net-guardia/src/model/direction.rs index 74874dd..403b19f 100644 --- a/net-guardia/src/model/direction.rs +++ b/net-guardia/src/model/direction.rs @@ -25,7 +25,7 @@ impl std::fmt::Display for Direction { } } -#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Hash, Debug)] #[serde(rename_all = "lowercase")] pub enum FlowDirection { Source, diff --git a/net-guardia/src/model/error/ebpf.rs b/net-guardia/src/model/error/ebpf.rs index f39002f..c59a1f7 100644 --- a/net-guardia/src/model/error/ebpf.rs +++ b/net-guardia/src/model/error/ebpf.rs @@ -64,6 +64,10 @@ traceable! { #[error("TX queue processing failed")] TXQueueError => tracing::Level::ERROR, + + #[no_source] + #[error("Flow statistics map key missing for ({direction}, {flow_direction}, {time_type})")] + FlowMapKeyMissing { direction: String, flow_direction: String, time_type: String } => tracing::Level::ERROR, } } diff --git a/net-guardia/src/model/error/ml.rs b/net-guardia/src/model/error/ml.rs index b342b45..8415541 100644 --- a/net-guardia/src/model/error/ml.rs +++ b/net-guardia/src/model/error/ml.rs @@ -19,5 +19,13 @@ traceable! { #[no_source] #[error("Failed to parse inference configuration: {reason}")] ConfigParseFailed { reason: String } => tracing::Level::ERROR, + + #[no_source] + #[error("Tracker mutex poisoned; skipping operation")] + TrackerLockPoisoned => tracing::Level::ERROR, + + #[no_source] + #[error("Inference flow buffers mutex poisoned; skipping operation")] + InferenceLockPoisoned => tracing::Level::ERROR, } } \ No newline at end of file diff --git a/net-guardia/src/model/error/mod.rs b/net-guardia/src/model/error/mod.rs index 94f94bf..3099af4 100644 --- a/net-guardia/src/model/error/mod.rs +++ b/net-guardia/src/model/error/mod.rs @@ -4,6 +4,7 @@ pub mod io; pub mod misc; pub mod ml; pub mod system; +pub mod rule; use serde::{Deserialize, Serialize}; @@ -12,6 +13,7 @@ use crate::model::error::http::HttpError; use crate::model::error::io::IOError; use crate::model::error::misc::MiscError; use crate::model::error::ml::MLError; +use crate::model::error::rule::RuleError; use crate::model::error::system::SystemError; #[derive(Clone, Debug, thiserror::Error, Serialize, Deserialize)] @@ -27,6 +29,8 @@ pub enum Error { #[error("{0}")] Misc(MiscError), #[error("{0}")] + Rule(RuleError), + #[error("{0}")] System(SystemError), } @@ -64,4 +68,10 @@ impl From for Error { fn from(error: MLError) -> Self { Self::ML(error) } +} + +impl From for Error { + fn from(error: RuleError) -> Self { + Self::Rule(error) + } } \ No newline at end of file diff --git a/net-guardia/src/model/error/rule.rs b/net-guardia/src/model/error/rule.rs new file mode 100644 index 0000000..6261c1b --- /dev/null +++ b/net-guardia/src/model/error/rule.rs @@ -0,0 +1,13 @@ +use macros::traceable; + +traceable! { + RuleError { + #[no_source] + #[error("BlockDatabase::new failed: {error}")] + BlockDatabaseFailed { error: String } => tracing::Level::ERROR, + + #[no_source] + #[error("Flow state missing for TCP key; skipping packet processing")] + FlowStateMissing => tracing::Level::ERROR, + } +} \ No newline at end of file diff --git a/net-guardia/src/model/log/ml.rs b/net-guardia/src/model/log/ml.rs index 76f69dd..f9d5c64 100644 --- a/net-guardia/src/model/log/ml.rs +++ b/net-guardia/src/model/log/ml.rs @@ -39,11 +39,11 @@ loggable! { #[error("Queue pair {queue_id} started successfully")] QueuePairStarted { queue_id: u32 } => tracing::Level::INFO, - #[error("ML models loaded - {info}")] + #[error("ML artifacts loaded - {info}")] ModelsLoaded { info: String } => tracing::Level::INFO, - #[error("Inference configuration loaded: {features} features, {attacks} attack types")] - ConfigLoaded { features: usize, attacks: usize } => tracing::Level::INFO, + #[error("Inference configuration loaded: {features} features")] + ConfigLoaded { features: usize } => tracing::Level::INFO, #[error("Inference completed: {total_flows} flows ({anomaly} anomaly, {benign} benign) in {duration_ms}ms ({throughput:.1} flows/s)")] InferenceCompleted { total_flows: usize, anomaly: usize, benign: usize, duration_ms: u32, throughput: f32 } => tracing::Level::INFO, @@ -68,5 +68,6 @@ loggable! { #[error("Failed to parse packet (length: {len})")] ParsePacketFailed { len: usize } => tracing::Level::INFO, + } } \ No newline at end of file diff --git a/net-guardia/src/model/log/mod.rs b/net-guardia/src/model/log/mod.rs index 9efbdfb..2e20b92 100644 --- a/net-guardia/src/model/log/mod.rs +++ b/net-guardia/src/model/log/mod.rs @@ -4,3 +4,4 @@ pub mod ml; pub mod system; pub mod misc; pub mod health; +pub mod rule; diff --git a/net-guardia/src/model/log/rule.rs b/net-guardia/src/model/log/rule.rs new file mode 100644 index 0000000..047df7c --- /dev/null +++ b/net-guardia/src/model/log/rule.rs @@ -0,0 +1,18 @@ +use macros::loggable; +use tracing; + +loggable! { + RuleLog { + #[error("Signature rules loaded: {count} patterns")] + RulesLoaded { count: usize } => tracing::Level::INFO, + + #[error("Signature rules disabled (no rules found in rules/)")] + RulesDisabled => tracing::Level::INFO, + + #[error("Signature match [{direction}]: {src}->{dst} sid={sid} \"{msg}\"")] + SignatureMatch { direction: String, src: String, dst: String, sid: u32, msg: String } => tracing::Level::WARN, + + #[error("Signature stats [{direction}]: active_flows={active_flows}, min_matches={min_matches}, match_counts: {counts}")] + SignatureStats { direction: String, active_flows: usize, min_matches: u32, counts: String } => tracing::Level::INFO, + } +} \ No newline at end of file diff --git a/net-guardia/src/model/log/system.rs b/net-guardia/src/model/log/system.rs index aa1d76e..22588c7 100644 --- a/net-guardia/src/model/log/system.rs +++ b/net-guardia/src/model/log/system.rs @@ -26,5 +26,6 @@ loggable! { #[error("Traffic logging mode enabled — writing packets to: {path}")] TrafficLoggingEnabled { path: String } => tracing::Level::INFO, + } } \ No newline at end of file diff --git a/net-guardia/src/model/ml_detection.rs b/net-guardia/src/model/ml_detection.rs index 970b331..bd5d9a1 100644 --- a/net-guardia/src/model/ml_detection.rs +++ b/net-guardia/src/model/ml_detection.rs @@ -12,26 +12,6 @@ pub struct ClipParams { pub lower: f64, pub upper: f64, } - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct AENormalization { - pub min: f64, - pub max: f64, - pub norm_max: f64, - pub mean: f64, - pub std: f64, - pub median: f64, - pub p90: f64, - pub p95: f64, - pub p99: f64, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PrecisionLevels { - pub threshold: f64, - pub precision: f64, -} - #[derive(Debug, Clone, Hash, Eq, PartialEq, Serialize, Deserialize)] pub struct FlowKey { pub src_ip: String, @@ -141,3 +121,41 @@ impl InferenceStats { pub struct EngineStats { pub active_flows: usize, } + +#[derive(Debug, Clone, Serialize)] +pub struct AlertMessage { + pub timestamp: u64, + pub flow_key: String, + pub src_ip: String, + pub dst_ip: String, + pub src_port: u16, + pub dst_port: u16, + pub protocol: u8, + pub is_attack: bool, + pub attack_type: Option, + pub confidence: f32, + pub ae_score: f32, +} + +impl AlertMessage { + pub fn from_detection_result(result: &DetectionResult) -> Self { + let timestamp = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time is after UNIX_EPOCH") + .as_secs(); + + Self { + timestamp, + flow_key: result.flow_key.clone(), + src_ip: result.flow_key_raw.src_ip.clone(), + dst_ip: result.flow_key_raw.dst_ip.clone(), + src_port: result.flow_key_raw.src_port, + dst_port: result.flow_key_raw.dst_port, + protocol: result.flow_key_raw.protocol, + is_attack: result.is_attack, + attack_type: result.attack_type.clone(), + confidence: result.confidence, + ae_score: result.ae_score, + } + } +} diff --git a/net-guardia/src/model/mod.rs b/net-guardia/src/model/mod.rs index a58d474..277a82e 100644 --- a/net-guardia/src/model/mod.rs +++ b/net-guardia/src/model/mod.rs @@ -8,3 +8,4 @@ pub mod log; pub mod time_type; pub mod ml_detection; pub mod health; +pub mod rule_detection; diff --git a/net-guardia/src/model/rule_detection.rs b/net-guardia/src/model/rule_detection.rs new file mode 100644 index 0000000..6c10bda --- /dev/null +++ b/net-guardia/src/model/rule_detection.rs @@ -0,0 +1,164 @@ +use std::net::IpAddr; +use std::time::Instant; +use protolens::{Packet as ProtoPacket, Task, TransProto}; + +#[derive(Debug, Clone)] +pub struct RuleMatch { + pub sid: u32, + pub msg: String, + pub src: String, + pub dst: String, +} + +pub struct ContentEntry { + pub pattern: Vec, + pub nocase: bool, + pub negated: bool, + pub has_distance: bool, + pub has_within: bool, + pub has_offset: bool, + pub has_depth: bool, + pub distance: i32, + pub within: i32, + pub offset: u16, + pub depth: u16, + + /// Target buffer ID (0 = raw; see buf module for named constants). + pub buffer: u8, +} + +/// Buffer IDs — must match the constants used in `build.rs` and `rule_engine.rs`. +/// 0 (Raw) has no named constant; it is the default when no sticky buffer is set. +pub mod buf { + pub const HTTP_URI: u8 = 1; + pub const HTTP_HEADER: u8 = 2; + pub const HTTP_CLIENT_BODY: u8 = 3; + pub const HTTP_SERVER_BODY: u8 = 4; + pub const HTTP_METHOD: u8 = 5; + pub const HTTP_USER_AGENT: u8 = 6; + pub const HTTP_HOST: u8 = 7; + pub const HTTP_STAT_CODE: u8 = 8; + pub const HTTP_COOKIE: u8 = 9; + pub const HTTP_RAW_URI: u8 = 10; + pub const TLS_SNI: u8 = 11; + pub const DNS_QUERY: u8 = 12; +} + +/// All application-layer fields extracted from a single payload chunk. +/// Empty `Vec` means the field was not present / not parsed. +pub struct ParsedPacket { + pub raw: Vec, + /// True when the payload is TLS Application Data (encrypted). + pub tls_app_data: bool, + /// True when this payload was identified as TLS (any record type, or port-based fallback). + pub is_tls: bool, + + pub http_uri: Vec, + pub http_raw_uri: Vec, + pub http_method: Vec, + pub http_host: Vec, + pub http_user_agent: Vec, + pub http_cookie: Vec, + pub http_header: Vec, + pub http_client_body: Vec, + + pub http_stat_code: Vec, + pub http_server_body: Vec, + + pub tls_sni: Vec, + + pub dns_query: Vec, +} + +impl ParsedPacket { + /// Return the byte slice for the given buffer ID. + pub fn buffer(&self, id: u8) -> &[u8] { + match id { + buf::HTTP_URI => &self.http_uri, + buf::HTTP_HEADER => &self.http_header, + buf::HTTP_CLIENT_BODY => &self.http_client_body, + buf::HTTP_SERVER_BODY => &self.http_server_body, + buf::HTTP_METHOD => &self.http_method, + buf::HTTP_USER_AGENT => &self.http_user_agent, + buf::HTTP_HOST => &self.http_host, + buf::HTTP_STAT_CODE => &self.http_stat_code, + buf::HTTP_COOKIE => &self.http_cookie, + buf::HTTP_RAW_URI => &self.http_raw_uri, + buf::TLS_SNI => &self.tls_sni, + buf::DNS_QUERY => &self.dns_query, + _ => &self.raw, + } + } +} + +pub struct RuleSig { + pub sid: u32, + /// 0=any, 1=to_server (ingress), 2=to_client (egress) + pub flow_dir: u8, + /// empty = any port + pub dports: Vec, + pub msg: String, + pub chain: Vec, +} + +#[derive(Clone)] +pub struct RawPkt { + pub trans_proto: TransProto, + pub sip: IpAddr, + pub dip: IpAddr, + pub sport: u16, + pub dport: u16, + pub seq: u32, + pub syn: bool, + pub fin: bool, + pub rst: bool, + pub payload: Vec, +} + +impl ProtoPacket for RawPkt { + fn trans_proto(&self) -> TransProto { self.trans_proto } + fn sip(&self) -> IpAddr { self.sip } + fn dip(&self) -> IpAddr { self.dip } + fn tu_sport(&self) -> u16 { self.sport } + fn tu_dport(&self) -> u16 { self.dport } + fn seq(&self) -> u32 { self.seq } + fn syn(&self) -> bool { self.syn } + fn fin(&self) -> bool { self.fin } + fn payload_len(&self) -> usize { self.payload.len() } + fn payload(&self) -> &[u8] { &self.payload } +} + +#[derive(Hash, Eq, PartialEq, Clone)] +pub struct FlowKey { + pub sip: IpAddr, + pub sport: u16, + pub dip: IpAddr, + pub dport: u16, +} + +impl FlowKey { + pub fn reversed(&self) -> Self { + Self { sip: self.dip, sport: self.dport, dip: self.sip, dport: self.sport } + } +} + +/// Application-layer protocol detected for a flow. +/// Mirrors Suricata's `flow->alproto` — once set, never cleared. +/// Only TLS is tracked for now; HTTP and DNS don't benefit from per-flow +/// skip logic because they're cheap to detect and should still be scanned. +#[derive(Clone, Copy, PartialEq, Eq, Default)] +pub enum FlowProto { + #[default] + Unknown = 0, + Tls = 1, +} + +pub struct FlowState { + pub task: Task, + pub last_seen: Instant, + pub fin_count: u8, + pub rst_seen: bool, + + /// Protocol identified on the first parseable chunk of this flow. + pub detected_proto: FlowProto, +} \ No newline at end of file diff --git a/net-guardia/src/model/time_type.rs b/net-guardia/src/model/time_type.rs index 83dce19..cdac296 100644 --- a/net-guardia/src/model/time_type.rs +++ b/net-guardia/src/model/time_type.rs @@ -1,6 +1,6 @@ use serde::{Deserialize, Serialize}; -#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Hash, Debug)] #[serde(rename_all = "lowercase")] pub enum TimeType { #[serde(rename = "1min")] diff --git a/net-guardia/src/utils/packet_parser.rs b/net-guardia/src/utils/packet_parser.rs index be4f2a9..a24fdb0 100644 --- a/net-guardia/src/utils/packet_parser.rs +++ b/net-guardia/src/utils/packet_parser.rs @@ -184,4 +184,4 @@ pub fn format_ipv6(addr: u128) -> String { bytes[14], bytes[15] ) -} +} \ No newline at end of file diff --git a/net-guardia/src/web/api/control/statistics.rs b/net-guardia/src/web/api/control/statistics.rs index b74f1a4..c0da53e 100644 --- a/net-guardia/src/web/api/control/statistics.rs +++ b/net-guardia/src/web/api/control/statistics.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use actix_web::{get, web, HttpRequest, HttpResponse, Responder, Scope}; +use macros::log; use crate::core::ebpf::statistics::Statistics; use crate::core::infrastructure::app_config::AppConfig; @@ -22,10 +23,13 @@ async fn get_ipv4_flow( statistics: web::Data>, ) -> impl Responder { let (direction, flow_direction, time_type) = path.into_inner(); - let flow_data = statistics - .get_ipv4_flow_data(direction, flow_direction, time_type) - .await; - HttpResponse::Ok().json(web::Json(flow_data)) + match statistics.get_ipv4_flow_data(direction, flow_direction, time_type).await { + Ok(flow_data) => HttpResponse::Ok().json(web::Json(flow_data)), + Err(e) => { + log!(e); + HttpResponse::InternalServerError().finish() + } + } } #[get("/get/ipv6/{direction}/{flow_direction}/{time_type}")] @@ -34,10 +38,13 @@ async fn get_ipv6_flow( statistics: web::Data>, ) -> impl Responder { let (direction, flow_direction, time_type) = path.into_inner(); - let flow_data = statistics - .get_ipv6_flow_data(direction, flow_direction, time_type) - .await; - HttpResponse::Ok().json(web::Json(flow_data)) + match statistics.get_ipv6_flow_data(direction, flow_direction, time_type).await { + Ok(flow_data) => HttpResponse::Ok().json(web::Json(flow_data)), + Err(e) => { + log!(e); + HttpResponse::InternalServerError().finish() + } + } } #[get("/websocket/ipv4/{direction}/{flow_direction}/{time_type}")] diff --git a/net-guardia/src/web/websocket/alert_websocket.rs b/net-guardia/src/web/websocket/alert_websocket.rs index f10e849..cdff3d4 100644 --- a/net-guardia/src/web/websocket/alert_websocket.rs +++ b/net-guardia/src/web/websocket/alert_websocket.rs @@ -4,7 +4,8 @@ use futures_util::StreamExt; use macros::log; use tokio::sync::broadcast; -use crate::core::infrastructure::ml_alert::{MLAlert, AlertMessage}; +use crate::core::infrastructure::ml_alert::MLAlert; +use crate::model::ml_detection::AlertMessage; use crate::model::error::http::HttpError; use crate::model::error::misc::MiscError; use crate::model::log::http::HttpLog; diff --git a/net-guardia/src/web/websocket/flow_websocket.rs b/net-guardia/src/web/websocket/flow_websocket.rs index 434ace5..1298b7c 100644 --- a/net-guardia/src/web/websocket/flow_websocket.rs +++ b/net-guardia/src/web/websocket/flow_websocket.rs @@ -158,9 +158,10 @@ async fn send_ipv4_flow_data( flow_direction: FlowDirection, time_type: TimeType, ) -> bool { - let flow_data = statistics - .get_ipv4_flow_data(direction, flow_direction, time_type) - .await; + let flow_data = match statistics.get_ipv4_flow_data(direction, flow_direction, time_type).await { + Ok(data) => data, + Err(e) => { log!(e); return false; } + }; match serde_json::to_string(&flow_data) { Ok(json) => session.text(json).await.is_ok(), Err(err) => { @@ -177,9 +178,10 @@ async fn send_ipv6_flow_data( flow_direction: FlowDirection, time_type: TimeType, ) -> bool { - let flow_data = statistics - .get_ipv6_flow_data(direction, flow_direction, time_type) - .await; + let flow_data = match statistics.get_ipv6_flow_data(direction, flow_direction, time_type).await { + Ok(data) => data, + Err(e) => { log!(e); return false; } + }; match serde_json::to_string(&flow_data) { Ok(json) => session.text(json).await.is_ok(), Err(err) => { diff --git a/net-guardia/static/artifacts/deep_autoencoder.onnx b/net-guardia/static/artifacts/deep_autoencoder.onnx new file mode 100644 index 0000000..4d13522 Binary files /dev/null and b/net-guardia/static/artifacts/deep_autoencoder.onnx differ diff --git a/net-guardia/static/artifacts/full_config.json b/net-guardia/static/artifacts/full_config.json new file mode 100644 index 0000000..e1b682c --- /dev/null +++ b/net-guardia/static/artifacts/full_config.json @@ -0,0 +1,251 @@ +{ + "created_at": "2026-04-21T10:38:25.314054", + "framework": "PyTorch", + "model": { + "lstm_deep_autoencoder": { + "file": "deep_autoencoder.onnx", + "type": "LSTM Autoencoder", + "input_dim": 27, + "hidden_size": 128, + "num_layers": 2, + "encoding_dim": 32, + "window_size": 10, + "ae_feature_names": [ + "flow_duration", + "fwd_packets", + "bwd_packets", + "fwd_bytes", + "bwd_bytes", + "flow_bytes_per_sec", + "flow_pkts_per_sec", + "fwd_win_bytes", + "bwd_win_bytes", + "fwd_pkt_len_mean", + "bwd_pkt_len_mean", + "fwd_iat_mean", + "bwd_iat_mean", + "flow_iat_mean", + "pkt_len_mean", + "dst_port", + "protocol", + "psh_flag_cnt", + "ack_flag_cnt", + "syn_flag_cnt", + "fin_flag_cnt", + "rst_flag_cnt", + "pkt_len_std", + "fwd_pkt_len_std", + "bwd_pkt_len_std", + "fwd_seg_size_min", + "fwd_act_data_pkts" + ], + "ae_threshold": 0.04424155503511429, + "ae_threshold_method": "mean+1std" + } + }, + "preprocessing": { + "ae_clip_params": { + "flow_duration": { + "lower": 13203.665, + "upper": 90041362.345 + }, + "fwd_packets": { + "lower": 2.0, + "upper": 1000.0 + }, + "bwd_packets": { + "lower": 0.0, + "upper": 1000.0 + }, + "fwd_bytes": { + "lower": 0.0, + "upper": 12828460.719999954 + }, + "bwd_bytes": { + "lower": 0.0, + "upper": 1782168.224999995 + }, + "flow_bytes_per_sec": { + "lower": 0.0, + "upper": 3787763.16328191 + }, + "flow_pkts_per_sec": { + "lower": 0.066635115985, + "upper": 2235.532986379341 + }, + "fwd_win_bytes": { + "lower": 0.0, + "upper": 65535.0 + }, + "bwd_win_bytes": { + "lower": 0.0, + "upper": 65535.0 + }, + "fwd_pkt_len_mean": { + "lower": 0.0, + "upper": 1448.0 + }, + "bwd_pkt_len_mean": { + "lower": 0.0, + "upper": 1415.023504181184 + }, + "fwd_iat_mean": { + "lower": 205.2674874874875, + "upper": 45017828.5025 + }, + "bwd_iat_mean": { + "lower": 0.0, + "upper": 45017992.675 + }, + "flow_iat_mean": { + "lower": 378.36644698085416, + "upper": 18008565.868 + }, + "pkt_len_mean": { + "lower": 0.0, + "upper": 1140.389183169216 + }, + "dst_port": { + "lower": 53.0, + "upper": 63575.0 + }, + "protocol": { + "lower": 6.0, + "upper": 17.0 + }, + "psh_flag_cnt": { + "lower": 0.0, + "upper": 3443.0 + }, + "ack_flag_cnt": { + "lower": 0.0, + "upper": 12277.0 + }, + "syn_flag_cnt": { + "lower": 0.0, + "upper": 3.0 + }, + "fin_flag_cnt": { + "lower": 0.0, + "upper": 2.0 + }, + "rst_flag_cnt": { + "lower": 0.0, + "upper": 3.0 + }, + "pkt_len_std": { + "lower": 0.0, + "upper": 724.1810679032936 + }, + "fwd_pkt_len_std": { + "lower": 0.0, + "upper": 677.6756844784483 + }, + "bwd_pkt_len_std": { + "lower": 0.0, + "upper": 723.5706185235359 + }, + "fwd_seg_size_min": { + "lower": 8.0, + "upper": 32.0 + }, + "fwd_act_data_pkts": { + "lower": 0.0, + "upper": 8950.0 + } + }, + "ae_scaler": { + "mean": [ + 5951288.750077261, + 59.11059332512558, + 64.43133138509893, + 222150.9194105303, + 43569.19774830678, + 172594.2514672874, + 144.20395486021135, + 17024.939887041404, + 16926.870109581538, + 178.341350807249, + 271.73465038994414, + 1866741.291807338, + 1837758.5476123434, + 887050.3497480378, + 275.91961765390863, + 15713.397431840604, + 12.181819994841344, + 60.86333020445825, + 263.60134098445224, + 0.5260804590671343, + 0.45693985380264684, + 0.2458391722751531, + 232.14679728383396, + 96.51238144611007, + 156.93610306173713, + 15.867236423296912, + 162.04478590251205 + ], + "std": [ + 13021032.695784755, + 189.6836628295549, + 186.13916771977108, + 1366100.0137844363, + 175343.2394873705, + 495919.246600428, + 332.6340993004248, + 28139.884220659886, + 27367.235623472247, + 346.83550077192035, + 422.80683501529063, + 5937390.371629928, + 6302828.0876488965, + 2607765.266228357, + 345.84585519370796, + 19782.572561725658, + 5.457574689789829, + 358.5939678553331, + 1365.024880946303, + 0.8897314831847252, + 0.8326227579585829, + 0.6757562381733568, + 261.91355522213735, + 145.97497775092612, + 209.49550260032348, + 9.755976125742986, + 953.2974129459069 + ], + "feature_names": [ + "flow_duration", + "fwd_packets", + "bwd_packets", + "fwd_bytes", + "bwd_bytes", + "flow_bytes_per_sec", + "flow_pkts_per_sec", + "fwd_win_bytes", + "bwd_win_bytes", + "fwd_pkt_len_mean", + "bwd_pkt_len_mean", + "fwd_iat_mean", + "bwd_iat_mean", + "flow_iat_mean", + "pkt_len_mean", + "dst_port", + "protocol", + "psh_flag_cnt", + "ack_flag_cnt", + "syn_flag_cnt", + "fin_flag_cnt", + "rst_flag_cnt", + "pkt_len_std", + "fwd_pkt_len_std", + "bwd_pkt_len_std", + "fwd_seg_size_min", + "fwd_act_data_pkts" + ] + }, + "post_scaling_clip": { + "min": -5.0, + "max": 5.0 + } + } +} \ No newline at end of file diff --git a/net-guardia/static/artifacts/inference_config.json b/net-guardia/static/artifacts/inference_config.json new file mode 100644 index 0000000..c742183 --- /dev/null +++ b/net-guardia/static/artifacts/inference_config.json @@ -0,0 +1,204 @@ +{ + "ae_feature_names": [ + "flow_duration", + "fwd_packets", + "bwd_packets", + "fwd_bytes", + "bwd_bytes", + "flow_bytes_per_sec", + "flow_pkts_per_sec", + "fwd_win_bytes", + "bwd_win_bytes", + "fwd_pkt_len_mean", + "bwd_pkt_len_mean", + "fwd_iat_mean", + "bwd_iat_mean", + "flow_iat_mean", + "pkt_len_mean", + "dst_port", + "protocol", + "psh_flag_cnt", + "ack_flag_cnt", + "syn_flag_cnt", + "fin_flag_cnt", + "rst_flag_cnt", + "pkt_len_std", + "fwd_pkt_len_std", + "bwd_pkt_len_std", + "fwd_seg_size_min", + "fwd_act_data_pkts" + ], + "ae_clip_params": { + "flow_duration": { + "lower": 13203.665, + "upper": 90041362.345 + }, + "fwd_packets": { + "lower": 2.0, + "upper": 1000.0 + }, + "bwd_packets": { + "lower": 0.0, + "upper": 1000.0 + }, + "fwd_bytes": { + "lower": 0.0, + "upper": 12828460.719999954 + }, + "bwd_bytes": { + "lower": 0.0, + "upper": 1782168.224999995 + }, + "flow_bytes_per_sec": { + "lower": 0.0, + "upper": 3787763.16328191 + }, + "flow_pkts_per_sec": { + "lower": 0.066635115985, + "upper": 2235.532986379341 + }, + "fwd_win_bytes": { + "lower": 0.0, + "upper": 65535.0 + }, + "bwd_win_bytes": { + "lower": 0.0, + "upper": 65535.0 + }, + "fwd_pkt_len_mean": { + "lower": 0.0, + "upper": 1448.0 + }, + "bwd_pkt_len_mean": { + "lower": 0.0, + "upper": 1415.023504181184 + }, + "fwd_iat_mean": { + "lower": 205.2674874874875, + "upper": 45017828.5025 + }, + "bwd_iat_mean": { + "lower": 0.0, + "upper": 45017992.675 + }, + "flow_iat_mean": { + "lower": 378.36644698085416, + "upper": 18008565.868 + }, + "pkt_len_mean": { + "lower": 0.0, + "upper": 1140.389183169216 + }, + "dst_port": { + "lower": 53.0, + "upper": 63575.0 + }, + "protocol": { + "lower": 6.0, + "upper": 17.0 + }, + "psh_flag_cnt": { + "lower": 0.0, + "upper": 3443.0 + }, + "ack_flag_cnt": { + "lower": 0.0, + "upper": 12277.0 + }, + "syn_flag_cnt": { + "lower": 0.0, + "upper": 3.0 + }, + "fin_flag_cnt": { + "lower": 0.0, + "upper": 2.0 + }, + "rst_flag_cnt": { + "lower": 0.0, + "upper": 3.0 + }, + "pkt_len_std": { + "lower": 0.0, + "upper": 724.1810679032936 + }, + "fwd_pkt_len_std": { + "lower": 0.0, + "upper": 677.6756844784483 + }, + "bwd_pkt_len_std": { + "lower": 0.0, + "upper": 723.5706185235359 + }, + "fwd_seg_size_min": { + "lower": 8.0, + "upper": 32.0 + }, + "fwd_act_data_pkts": { + "lower": 0.0, + "upper": 8950.0 + } + }, + "ae_scaler_mean": [ + 5951288.750077261, + 59.11059332512558, + 64.43133138509893, + 222150.9194105303, + 43569.19774830678, + 172594.2514672874, + 144.20395486021135, + 17024.939887041404, + 16926.870109581538, + 178.341350807249, + 271.73465038994414, + 1866741.291807338, + 1837758.5476123434, + 887050.3497480378, + 275.91961765390863, + 15713.397431840604, + 12.181819994841344, + 60.86333020445825, + 263.60134098445224, + 0.5260804590671343, + 0.45693985380264684, + 0.2458391722751531, + 232.14679728383396, + 96.51238144611007, + 156.93610306173713, + 15.867236423296912, + 162.04478590251205 + ], + "ae_scaler_std": [ + 13021032.695784755, + 189.6836628295549, + 186.13916771977108, + 1366100.0137844363, + 175343.2394873705, + 495919.246600428, + 332.6340993004248, + 28139.884220659886, + 27367.235623472247, + 346.83550077192035, + 422.80683501529063, + 5937390.371629928, + 6302828.0876488965, + 2607765.266228357, + 345.84585519370796, + 19782.572561725658, + 5.457574689789829, + 358.5939678553331, + 1365.024880946303, + 0.8897314831847252, + 0.8326227579585829, + 0.6757562381733568, + 261.91355522213735, + 145.97497775092612, + 209.49550260032348, + 9.755976125742986, + 953.2974129459069 + ], + "ae_post_clip_min": -5.0, + "ae_post_clip_max": 5.0, + "ae_threshold": 0.04424155503511429, + "ae_threshold_method": "mean+1std", + "window_size": 10 +} \ No newline at end of file diff --git a/net-guardia/static/artifacts/info.txt b/net-guardia/static/artifacts/info.txt new file mode 100644 index 0000000..6515edb --- /dev/null +++ b/net-guardia/static/artifacts/info.txt @@ -0,0 +1 @@ +// this's ML model folder, so you can use Trainer output model \ No newline at end of file diff --git a/net-guardia/static/geo/info.txt b/net-guardia/static/geo/info.txt new file mode 100644 index 0000000..1243eb9 --- /dev/null +++ b/net-guardia/static/geo/info.txt @@ -0,0 +1,2 @@ +// this's GeoLite folder, so you can download the open rule from https://github.com/P3TERX/GeoLite.mmdb/raw/download/GeoLite2-City.mmdb or https://git.io/GeoLite2-City.mmdb +// you can access to https://github.com/P3TERX/GeoLite.mmdb for origin repo \ No newline at end of file diff --git a/net-guardia/static/rules/info.txt b/net-guardia/static/rules/info.txt new file mode 100644 index 0000000..3d14aa6 --- /dev/null +++ b/net-guardia/static/rules/info.txt @@ -0,0 +1 @@ +// this's open rule folder, so you can download the open rule from https://rules.emergingthreats.net/open/suricata-8.0/emerging.rules.tar.gz