From fb1dc93adf8fb51a410bcf852efab27bf856f382 Mon Sep 17 00:00:00 2001 From: DaLaw2 Date: Tue, 7 Apr 2026 15:43:31 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20v10=20model=20adaptation=20=E2=80=94=20?= =?UTF-8?q?aggregator=20fix,=20test=20updates,=20submodule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix aggregator score/threshold mismatch (ae_score vs anomaly_threshold) - Add class_min_confidence to MLInferenceConfig - Exploitation: min_detections=1 (single-shot attacks) - Update tests: Cryptomining → Exploitation/Bot/DNS Tunneling - Remove unused threshold field from DetectionResult - Update trainer submodule to v10 Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.lock | 126 +++++++++++++++--- autoresearch-results.tsv | 6 + models/inference_config.json | 5 +- net-guardia-frontend | 2 +- net-guardia-trainer | 2 +- .../src/adapter/persistence/repository.rs | 3 +- .../src/adapter/websocket/alert_websocket.rs | 2 +- .../src/adapter/websocket/drop_websocket.rs | 2 +- .../src/adapter/websocket/health_websocket.rs | 2 +- net-guardia/src/core/acl_service.rs | 2 +- net-guardia/src/core/auth/jwt.rs | 3 +- net-guardia/src/core/ebpf/protocol_filter.rs | 2 +- net-guardia/src/core/ebpf/xsk_manager.rs | 4 +- net-guardia/src/core/ml/aggregator.rs | 22 ++- net-guardia/src/core/ml/alert.rs | 2 +- net-guardia/src/core/ml/engine.rs | 14 +- net-guardia/src/core/ml/feature_extractor.rs | 6 +- net-guardia/src/core/ml/flow_tracker.rs | 2 +- net-guardia/src/core/ml/inference.rs | 2 - net-guardia/src/core/ml/model_loader.rs | 11 +- net-guardia/src/core/ml/model_watcher.rs | 17 +-- net-guardia/src/core/system.rs | 14 +- net-guardia/src/infrastructure/app_config.rs | 2 +- .../src/infrastructure/app_services.rs | 4 +- net-guardia/src/infrastructure/health.rs | 2 +- net-guardia/src/infrastructure/http_server.rs | 2 +- .../src/infrastructure/service_factory.rs | 6 +- net-guardia/src/infrastructure/statistics.rs | 4 +- net-guardia/src/interface/port/api_key.rs | 2 +- .../src/model/detection/ml_detection.rs | 1 - net-guardia/src/model/system/config.rs | 6 + 31 files changed, 201 insertions(+), 79 deletions(-) create mode 100644 autoresearch-results.tsv diff --git a/Cargo.lock b/Cargo.lock index ebee79c..ca987df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,7 +11,7 @@ dependencies = [ "actix-macros", "actix-rt", "actix_derive", - "bitflags", + "bitflags 2.11.0", "bytes", "crossbeam-channel", "futures-core", @@ -33,7 +33,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" dependencies = [ - "bitflags", + "bitflags 2.11.0", "bytes", "futures-core", "futures-sink", @@ -70,7 +70,7 @@ dependencies = [ "actix-service", "actix-utils", "base64", - "bitflags", + "bitflags 2.11.0", "brotli", "bytes", "bytestring", @@ -421,6 +421,15 @@ dependencies = [ "object 0.37.3", ] +[[package]] +name = "arc-swap" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +dependencies = [ + "rustversion", +] + [[package]] name = "argon2" version = "0.5.3" @@ -470,7 +479,7 @@ checksum = "d18bc4e506fbb85ab7392ed993a7db4d1a452c71b75a246af4a80ab8c9d2dd50" dependencies = [ "assert_matches", "aya-obj", - "bitflags", + "bitflags 2.11.0", "bytes", "libc", "log", @@ -607,7 +616,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -636,6 +645,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.0" @@ -1778,6 +1793,26 @@ dependencies = [ "which", ] +[[package]] +name = "inotify" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd168d97690d0b8c412d6b6c10360277f4d7ee495c5d0d5d5fe0854923255cc" +dependencies = [ + "bitflags 1.3.2", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.4" @@ -1787,6 +1822,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.12.0" @@ -1892,6 +1936,26 @@ dependencies = [ "simple_asn1", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "kstring" version = "2.0.2" @@ -1988,7 +2052,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", "plain", "redox_syscall 0.7.3", @@ -2281,6 +2345,7 @@ dependencies = [ "actix-web", "actix-ws", "aes-gcm", + "arc-swap", "argon2", "async-trait", "aya", @@ -2294,6 +2359,7 @@ dependencies = [ "dotenvy", "futures-util", "hkdf", + "hmac", "ipnetwork", "jsonwebtoken", "lettre", @@ -2304,6 +2370,7 @@ dependencies = [ "maxminddb", "mime_guess", "network-types", + "notify", "parking_lot", "r2d2", "r2d2_sqlite", @@ -2356,7 +2423,7 @@ version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -2390,6 +2457,33 @@ dependencies = [ "nom 8.0.0", ] +[[package]] +name = "notify" +version = "7.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c533b4c39709f9ba5005d8002048266593c1cfaf3c5f0739d5b8ab0c6c504009" +dependencies = [ + "bitflags 2.11.0", + "filetime", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.52.0", +] + +[[package]] +name = "notify-types" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585d3cb5e12e01aed9e8a1f70d5c6b5e86fe2a6e48fc8cd0b3e0b8df6f6eb174" +dependencies = [ + "instant", +] + [[package]] name = "ntapi" version = "0.4.3" @@ -2479,7 +2573,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -2980,7 +3074,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -2989,7 +3083,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -3085,7 +3179,7 @@ version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e34486da88d8e051c7c0e23c3f15fd806ea8546260aa2fec247e97242ec143" dependencies = [ - "bitflags", + "bitflags 2.11.0", "fallible-iterator", "fallible-streaming-iterator", "hashlink", @@ -3162,7 +3256,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys", @@ -3790,7 +3884,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.11.0", "bytes", "futures-util", "http 1.4.0", @@ -4315,7 +4409,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap", "semver", @@ -4655,7 +4749,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.0", "indexmap", "log", "serde", @@ -4707,7 +4801,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1fef46e3505c5055082f52ada0a7f8e5dcaebdbb9eccf8e978c32382c159270" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "libc", "libxdp-sys", diff --git a/autoresearch-results.tsv b/autoresearch-results.tsv new file mode 100644 index 0000000..5e17a55 --- /dev/null +++ b/autoresearch-results.tsv @@ -0,0 +1,6 @@ +# metric_direction: higher_is_better +iteration commit metric delta guard status description +0 4216906 91.3 0.0 - baseline v4 model — 91.3% accuracy, Normal FPR 29.3% +1 4216906 77.9 -13.4 - keep v5: 10 classes, danger weights, FPR 3.0% (from 29.3%), macro F1 0.804 +2 4216906 83.4 +5.5 - keep v6: class_loss=1.0, focal_gamma=2.0, patience=40. Macro F1 0.848, FPR 3.0% +3 4216906 86.4 +3.0 - keep v7: wider model 384→256→128 (638K), Recon+0.104, Crypto collapsed to 0 diff --git a/models/inference_config.json b/models/inference_config.json index 6d705ff..9204c8e 100644 --- a/models/inference_config.json +++ b/models/inference_config.json @@ -266,7 +266,7 @@ "1": "Brute Force", "2": "C2 Communication", "3": "DNS Tunneling", - "4": "DoS\/DDoS", + "4": "DoS/DDoS", "5": "Exploitation", "6": "Malware", "7": "Normal", @@ -313,5 +313,6 @@ "bwd_iat_std": 1.5, "fwd_bwd_bytes_ratio": 2.0, "iat_cv": 2.0 - } + }, + "class_min_confidence": 0.4 } \ No newline at end of file diff --git a/net-guardia-frontend b/net-guardia-frontend index dbe5342..8883b04 160000 --- a/net-guardia-frontend +++ b/net-guardia-frontend @@ -1 +1 @@ -Subproject commit dbe5342900202407fa47ad2e8062326cb0608846 +Subproject commit 8883b04ac722c8c3da29cc60adec0391af8b0b42 diff --git a/net-guardia-trainer b/net-guardia-trainer index 3447472..dea59f2 160000 --- a/net-guardia-trainer +++ b/net-guardia-trainer @@ -1 +1 @@ -Subproject commit 344747288ec264a83997299a636959c4fe5989ef +Subproject commit dea59f289635445fe63e69db8345df2f444fb9f1 diff --git a/net-guardia/src/adapter/persistence/repository.rs b/net-guardia/src/adapter/persistence/repository.rs index 348e4ab..0e7c6d8 100644 --- a/net-guardia/src/adapter/persistence/repository.rs +++ b/net-guardia/src/adapter/persistence/repository.rs @@ -126,8 +126,7 @@ impl Database { use std::fmt::Write; type HmacSha256 = Hmac; - let mut mac = HmacSha256::new_from_slice(&self.api_key_hmac) - .unwrap_or_else(|_| unreachable!()); + let mut mac = HmacSha256::new_from_slice(&self.api_key_hmac).unwrap_or_else(|_| unreachable!()); mac.update(raw_key.as_bytes()); let result = mac.finalize().into_bytes(); diff --git a/net-guardia/src/adapter/websocket/alert_websocket.rs b/net-guardia/src/adapter/websocket/alert_websocket.rs index f3fba72..555f235 100644 --- a/net-guardia/src/adapter/websocket/alert_websocket.rs +++ b/net-guardia/src/adapter/websocket/alert_websocket.rs @@ -5,10 +5,10 @@ use macros::log; use tokio::sync::broadcast; use crate::core::ml::alert::MLAlert; +use crate::model::detection::ml_detection::AlertMessage; use crate::model::error::http::HttpError; use crate::model::error::misc::MiscError; use crate::model::log::http::HttpLog; -use crate::model::detection::ml_detection::AlertMessage; pub async fn websocket_alert(req: HttpRequest, body: web::Payload, ai: web::Data) -> Result { let (response, session, msg_stream) = handle(&req, body)?; diff --git a/net-guardia/src/adapter/websocket/drop_websocket.rs b/net-guardia/src/adapter/websocket/drop_websocket.rs index 66b9294..9a390d7 100644 --- a/net-guardia/src/adapter/websocket/drop_websocket.rs +++ b/net-guardia/src/adapter/websocket/drop_websocket.rs @@ -5,10 +5,10 @@ use macros::log; use tokio::sync::broadcast; use crate::core::ebpf::drop_monitor::DropMonitor; -use crate::model::monitoring::drop_event::DropEventMessage; use crate::model::error::http::HttpError; use crate::model::error::misc::MiscError; use crate::model::log::http::HttpLog; +use crate::model::monitoring::drop_event::DropEventMessage; pub async fn websocket_drops( req: HttpRequest, diff --git a/net-guardia/src/adapter/websocket/health_websocket.rs b/net-guardia/src/adapter/websocket/health_websocket.rs index 91937a1..feb9755 100644 --- a/net-guardia/src/adapter/websocket/health_websocket.rs +++ b/net-guardia/src/adapter/websocket/health_websocket.rs @@ -7,8 +7,8 @@ use tokio::sync::broadcast; use crate::infrastructure::health::SystemHealth; use crate::model::error::http::HttpError; use crate::model::error::misc::MiscError; -use crate::model::system::health::SystemHealthMetrics; use crate::model::log::http::HttpLog; +use crate::model::system::health::SystemHealthMetrics; pub async fn websocket_system_health( req: HttpRequest, diff --git a/net-guardia/src/core/acl_service.rs b/net-guardia/src/core/acl_service.rs index ced498d..55a6c7a 100644 --- a/net-guardia/src/core/acl_service.rs +++ b/net-guardia/src/core/acl_service.rs @@ -7,9 +7,9 @@ use crate::interface::port::repository::RepositoryPort; use crate::model::monitoring::direction::FlowDirection; use macros::log; +use crate::model::access_control::list_type::ListType; use crate::model::error::Error; use crate::model::error::ebpf::EbpfError; -use crate::model::access_control::list_type::ListType; /// Domain service that coordinates ACL changes between DB persistence and eBPF data plane. /// Atomic write: eBPF first, then DB. If DB fails, rollback eBPF. diff --git a/net-guardia/src/core/auth/jwt.rs b/net-guardia/src/core/auth/jwt.rs index d7c2b7e..a4390b4 100644 --- a/net-guardia/src/core/auth/jwt.rs +++ b/net-guardia/src/core/auth/jwt.rs @@ -3,9 +3,9 @@ use std::sync::Arc; use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header, Validation, decode, encode, errors::ErrorKind}; use crate::interface::port::secret_store::SecretStorePort; -use crate::model::identity::auth::Claims; use crate::model::error::Error; use crate::model::error::auth::AuthError; +use crate::model::identity::auth::Claims; pub struct JwtService { encoding_key: EncodingKey, @@ -135,5 +135,4 @@ mod tests { let result = jwt2.validate_token(&token); assert!(result.is_err()); } - } diff --git a/net-guardia/src/core/ebpf/protocol_filter.rs b/net-guardia/src/core/ebpf/protocol_filter.rs index 4fa292d..c7ecf4a 100644 --- a/net-guardia/src/core/ebpf/protocol_filter.rs +++ b/net-guardia/src/core/ebpf/protocol_filter.rs @@ -8,9 +8,9 @@ use common::model::ip_address::{AddrPortV4, AddrPortV6, IPv4, IPv6}; use common::model::placeholder::PlaceHolder; use tokio::sync::RwLock; +use crate::model::access_control::ip_address::NativeConvert; use crate::model::error::Error; use crate::model::error::ebpf::EbpfError; -use crate::model::access_control::ip_address::NativeConvert; pub struct ProtocolFilter { ipv4_http_service: RwLock>, diff --git a/net-guardia/src/core/ebpf/xsk_manager.rs b/net-guardia/src/core/ebpf/xsk_manager.rs index 741a9ae..7ccd0d3 100644 --- a/net-guardia/src/core/ebpf/xsk_manager.rs +++ b/net-guardia/src/core/ebpf/xsk_manager.rs @@ -20,12 +20,12 @@ use crate::core::ebpf::dns_filter::DnsFilter; use crate::core::ml::engine::Engine; use crate::core::ml::flow_tracker::FlowTracker; use crate::infrastructure::app_config::AppConfig; -use crate::model::monitoring::direction::Direction; -use crate::model::system::config::NetworkConfig; use crate::model::error::Error; use crate::model::error::ebpf::EbpfError; use crate::model::error::system::SystemError; use crate::model::log::ebpf::EbpfLog; +use crate::model::monitoring::direction::Direction; +use crate::model::system::config::NetworkConfig; use crate::utils::packet_parser::parse_packet; /// Pre-allocated buffer pool to avoid per-packet malloc. diff --git a/net-guardia/src/core/ml/aggregator.rs b/net-guardia/src/core/ml/aggregator.rs index afe81d5..15b76a1 100644 --- a/net-guardia/src/core/ml/aggregator.rs +++ b/net-guardia/src/core/ml/aggregator.rs @@ -29,10 +29,10 @@ impl AttackAggregator { // Per-attack-type adaptive min_detections: // DoS/DDoS: high frequency, need more confirmations to avoid alert storms - // C2/Bot/DNS Tunneling: low frequency, alert on first detection + // C2/Bot/DNS Tunneling/Exploitation: low frequency or single-shot, alert immediately let effective_min = match attack_type { Some("DoS/DDoS") => self.min_detections.saturating_mul(2).max(1), - Some("C2 Communication") | Some("Bot") | Some("DNS Tunneling") => 1, + Some("C2 Communication") | Some("Bot") | Some("DNS Tunneling") | Some("Exploitation") => 1, _ => self.min_detections, }; @@ -99,10 +99,24 @@ mod tests { } #[test] - fn cryptomining_alerts_on_first_detection() { + fn exploitation_alerts_on_first_detection() { let mut agg = AttackAggregator::new(60, 3); let key = test_key(); - assert!(agg.should_alert(&key, 5.0, 1.0, Some("Cryptomining"))); + assert!(agg.should_alert(&key, 5.0, 1.0, Some("Exploitation"))); + } + + #[test] + fn bot_alerts_on_first_detection() { + let mut agg = AttackAggregator::new(60, 3); + let key = test_key(); + assert!(agg.should_alert(&key, 5.0, 1.0, Some("Bot"))); + } + + #[test] + fn dns_tunneling_alerts_on_first_detection() { + let mut agg = AttackAggregator::new(60, 3); + let key = test_key(); + assert!(agg.should_alert(&key, 5.0, 1.0, Some("DNS Tunneling"))); } #[test] diff --git a/net-guardia/src/core/ml/alert.rs b/net-guardia/src/core/ml/alert.rs index 77e0f98..e2e0aeb 100644 --- a/net-guardia/src/core/ml/alert.rs +++ b/net-guardia/src/core/ml/alert.rs @@ -2,8 +2,8 @@ use macros::log; use tokio::sync::broadcast; use crate::model::config::constants::ML_ALERT_CHANNEL_CAPACITY; -use crate::model::log::ml::MLLog; use crate::model::detection::ml_detection::{AlertMessage, DetectionResult}; +use crate::model::log::ml::MLLog; pub struct MLAlert { broadcast_tx: broadcast::Sender, diff --git a/net-guardia/src/core/ml/engine.rs b/net-guardia/src/core/ml/engine.rs index 451930a..f7e1aaf 100644 --- a/net-guardia/src/core/ml/engine.rs +++ b/net-guardia/src/core/ml/engine.rs @@ -7,17 +7,17 @@ use tokio::sync::oneshot; use tokio::time::interval; use super::aggregator::AttackAggregator; -use crate::model::system::config::MLInferenceConfig; use super::drift_detector::DriftDetector; use super::flow_tracker::{FlowData, FlowTracker}; use super::inference::Inference; use super::model_loader::MLModels; use super::traffic_logger::TrafficLogger; use crate::model::detection::flow_features::FlowFeatures; +use crate::model::system::config::MLInferenceConfig; use super::alert::MLAlert; -use crate::model::log::ml::MLLog; use crate::model::detection::ml_detection::{EngineConfig, InferenceStats}; +use crate::model::log::ml::MLLog; /// Per-queue tracker. With symmetric hash in eBPF, both directions of a flow /// land on the same queue, so per-queue trackers correctly see bidirectional flows. @@ -168,9 +168,9 @@ impl Engine { let mut t = tracker.lock(); total_count += t.flow_count(); all_flows.extend( - t.get_uninferred_flows() - .into_iter() - .filter(|flow| flow.packet_count() >= Self::effective_min_packets(&flow.flow_key, self.min_packets)), + t.get_uninferred_flows().into_iter().filter(|flow| { + flow.packet_count() >= Self::effective_min_packets(&flow.flow_key, self.min_packets) + }), ); // lock released here } @@ -246,8 +246,8 @@ impl Engine { if result.is_attack { let should_alert = aggregator.should_alert( &result.flow_key_raw, - result.ae_score, - result.threshold, + result.confidence, + self.inference_pipeline.config.class_min_confidence, result.attack_type.as_deref(), ); diff --git a/net-guardia/src/core/ml/feature_extractor.rs b/net-guardia/src/core/ml/feature_extractor.rs index 32140e7..cbad671 100644 --- a/net-guardia/src/core/ml/feature_extractor.rs +++ b/net-guardia/src/core/ml/feature_extractor.rs @@ -120,7 +120,7 @@ struct PrecomputedStats { idle_mean: f64, idle_std: f64, - // Phase 2: new features for C2/Cryptomining detection + // Phase 2: new features for C2/Bot detection fwd_bwd_bytes_ratio: f64, fwd_iat_skewness: f64, } @@ -207,7 +207,7 @@ impl PrecomputedStats { let (idle_max, idle_min, idle_mean, idle_std) = compute_stats(&flow.idle_periods.iter().map(|&x| x as f64).collect::>()); - // Phase 2: new features for C2/Cryptomining detection + // Phase 2: new features for C2/Bot detection let fwd_bwd_bytes_ratio = safe_div(fwd_total_bytes, fwd_total_bytes + bwd_total_bytes); let fwd_iat_skewness = compute_bowley_skewness(&fwd_iats); @@ -374,7 +374,7 @@ impl PrecomputedStats { "bwd_iat_std" => self.bwd_iat_std, "flow_iat_std" => self.flow_iat_std, - // Phase 2: new features for C2/Cryptomining detection + // Phase 2: new features for C2/Bot detection "fwd_bwd_bytes_ratio" => self.fwd_bwd_bytes_ratio, "pkt_len_variance" => self.all_len_std * self.all_len_std, "fwd_iat_skewness" => self.fwd_iat_skewness, diff --git a/net-guardia/src/core/ml/flow_tracker.rs b/net-guardia/src/core/ml/flow_tracker.rs index 52cfc4a..b56d61a 100644 --- a/net-guardia/src/core/ml/flow_tracker.rs +++ b/net-guardia/src/core/ml/flow_tracker.rs @@ -7,8 +7,8 @@ use crate::model::config::constants::{ FLOW_BULK_MIN_BYTES, FLOW_BULK_MIN_PACKETS, FLOW_IDLE_THRESHOLD_US, FLOW_IDLE_TIMEOUT_US, FLOW_MAX_PACKETS_PER_DIRECTION, FLOW_MAX_PERIODS, FLOW_TERMINATED_TIMEOUT_US, }; -use crate::model::monitoring::direction::Direction; use crate::model::detection::ml_detection::{BulkState, FlowKey, PacketData}; +use crate::model::monitoring::direction::Direction; use crate::model::monitoring::user_packet::UserPacket; #[derive(Debug, Clone)] diff --git a/net-guardia/src/core/ml/inference.rs b/net-guardia/src/core/ml/inference.rs index 4639eb6..ec69762 100644 --- a/net-guardia/src/core/ml/inference.rs +++ b/net-guardia/src/core/ml/inference.rs @@ -295,7 +295,6 @@ impl Inference { ae_score, anomaly_score, c2_score, - threshold: self.config.anomaly_threshold, packet_count: flow.packet_count() as u64, flow_duration_us: flow.duration_us(), }; @@ -311,7 +310,6 @@ impl Inference { ae_score, anomaly_score, c2_score, - threshold: self.config.anomaly_threshold, packet_count: flow.packet_count() as u64, flow_duration_us: flow.duration_us(), } diff --git a/net-guardia/src/core/ml/model_loader.rs b/net-guardia/src/core/ml/model_loader.rs index 394bae5..baa7ab4 100644 --- a/net-guardia/src/core/ml/model_loader.rs +++ b/net-guardia/src/core/ml/model_loader.rs @@ -40,11 +40,7 @@ impl MLModels { fn loader(model: &str, features: usize, batch_size: usize) -> Result { let model_path = PathBuf::from("models").join(model); - log!(MLLog::ModelLoading( - model.to_string(), - features, - batch_size - )); + log!(MLLog::ModelLoading(model.to_string(), features, batch_size)); let start = Instant::now(); @@ -71,6 +67,9 @@ impl MLModels { let inputs = model.model().inputs.len(); let outputs = model.model().outputs.len(); - format!("{name}: inputs: {inputs}, outputs: {outputs}, batch_size: {}", self.batch_size) + format!( + "{name}: inputs: {inputs}, outputs: {outputs}, batch_size: {}", + self.batch_size + ) } } diff --git a/net-guardia/src/core/ml/model_watcher.rs b/net-guardia/src/core/ml/model_watcher.rs index a131cdf..f66f2ae 100644 --- a/net-guardia/src/core/ml/model_watcher.rs +++ b/net-guardia/src/core/ml/model_watcher.rs @@ -81,19 +81,14 @@ impl ModelWatcher { Ok(()) } - fn spawn_watcher( - models_dir: PathBuf, - tx: mpsc::Sender<()>, - ) -> Result { + fn spawn_watcher(models_dir: PathBuf, tx: mpsc::Sender<()>) -> Result { let mut watcher = notify::recommended_watcher(move |res: Result| { if let Ok(event) = res { - let dominated = matches!( - event.kind, - EventKind::Create(_) | EventKind::Modify(_) - ); - let has_onnx = event.paths.iter().any(|p| { - p.extension().is_some_and(|ext| ext == "onnx") - }); + let dominated = matches!(event.kind, EventKind::Create(_) | EventKind::Modify(_)); + let has_onnx = event + .paths + .iter() + .any(|p| p.extension().is_some_and(|ext| ext == "onnx")); if dominated && has_onnx { let _ = tx.blocking_send(()); } diff --git a/net-guardia/src/core/system.rs b/net-guardia/src/core/system.rs index 1b5177f..0fd4d86 100644 --- a/net-guardia/src/core/system.rs +++ b/net-guardia/src/core/system.rs @@ -464,8 +464,18 @@ mod tests { } #[test] - fn normalize_cryptomining() { - assert_eq!(System::normalize_attack_type("Cryptomining"), "cryptomining"); + fn normalize_malware() { + assert_eq!(System::normalize_attack_type("Malware"), "threat_detected"); + } + + #[test] + fn normalize_bot() { + assert_eq!(System::normalize_attack_type("Bot"), "threat_detected"); + } + + #[test] + fn normalize_dns_tunneling() { + assert_eq!(System::normalize_attack_type("DNS Tunneling"), "threat_detected"); } #[test] diff --git a/net-guardia/src/infrastructure/app_config.rs b/net-guardia/src/infrastructure/app_config.rs index 7b7cd8c..0c2d623 100644 --- a/net-guardia/src/infrastructure/app_config.rs +++ b/net-guardia/src/infrastructure/app_config.rs @@ -1,7 +1,7 @@ use crate::adapter::persistence::Database; -use crate::model::system::config::{HttpConfig, InferenceConfig, MiscConfig, NetworkConfig, PipelineConfig}; use crate::model::error::Error; use crate::model::error::system::SystemError; +use crate::model::system::config::{HttpConfig, InferenceConfig, MiscConfig, NetworkConfig, PipelineConfig}; pub struct AppConfig { pub http: HttpConfig, diff --git a/net-guardia/src/infrastructure/app_services.rs b/net-guardia/src/infrastructure/app_services.rs index 54494c2..eaddf4d 100644 --- a/net-guardia/src/infrastructure/app_services.rs +++ b/net-guardia/src/infrastructure/app_services.rs @@ -6,7 +6,6 @@ use macros::log; use tokio::sync::oneshot; use crate::core::ml::alert::MLAlert; -use crate::model::system::config::MLInferenceConfig; use crate::core::ml::drift_detector::DriftDetector; use crate::core::ml::engine::Engine; use crate::core::ml::model_loader::MLModels; @@ -15,11 +14,12 @@ use crate::infrastructure::app_config::AppConfig; use crate::infrastructure::health::SystemHealth; use crate::infrastructure::statistics::FlowStatistics; use crate::model::detection::flow_features::FlowFeatures; +use crate::model::detection::ml_detection::EngineConfig; use crate::model::error::Error; use crate::model::error::misc::MiscError; use crate::model::error::system::SystemError; use crate::model::log::system::SystemLog; -use crate::model::detection::ml_detection::EngineConfig; +use crate::model::system::config::MLInferenceConfig; /// Application-level service orchestrator. /// Holds all runtime services (health monitoring, ML inference, flow statistics) diff --git a/net-guardia/src/infrastructure/health.rs b/net-guardia/src/infrastructure/health.rs index caec10d..3d3db39 100644 --- a/net-guardia/src/infrastructure/health.rs +++ b/net-guardia/src/infrastructure/health.rs @@ -8,11 +8,11 @@ use tokio::time::interval; use crate::infrastructure::app_config::AppConfig; use crate::model::error::Error; +use crate::model::log::health::Health; use crate::model::system::health::{ ConfiguredNetworkStats, CpuCoreInfo, CpuDetails, LoadAverage, MemoryUsage, NetworkStats, SystemHealthMetrics, SystemHealthStatus, SystemInfo, }; -use crate::model::log::health::Health; pub struct SystemHealth { system: RwLock, diff --git a/net-guardia/src/infrastructure/http_server.rs b/net-guardia/src/infrastructure/http_server.rs index 9b80064..0033b26 100644 --- a/net-guardia/src/infrastructure/http_server.rs +++ b/net-guardia/src/infrastructure/http_server.rs @@ -17,7 +17,6 @@ use crate::core::auth::setup_guard::{SetupCompleteFlag, SetupGuard}; use crate::core::config_service::ConfigService; use crate::core::dns_filter_service::DnsFilterService; use crate::core::ebpf::EbpfServices; -use crate::model::system::config::MLInferenceConfig; use crate::core::notification_service::NotificationService; use crate::core::playbook_service::PlaybookService; use crate::core::rate_limit_service::RateLimitService; @@ -32,6 +31,7 @@ use crate::model::config::constants::HTTP_FALLBACK_PORT; use crate::model::error::Error; use crate::model::error::http::HttpError; use crate::model::log::http::HttpLog; +use crate::model::system::config::MLInferenceConfig; use macros::log; /// Shared flag: true when all services (eBPF, ML, SOAR) are fully initialized. diff --git a/net-guardia/src/infrastructure/service_factory.rs b/net-guardia/src/infrastructure/service_factory.rs index bcd31ca..8b7d24e 100644 --- a/net-guardia/src/infrastructure/service_factory.rs +++ b/net-guardia/src/infrastructure/service_factory.rs @@ -19,7 +19,6 @@ use crate::core::config_service::ConfigService; use crate::core::dns_filter_service::DnsFilterService; use crate::core::ebpf::EbpfServices; use crate::core::email::scheduler::ReportScheduler; -use crate::model::system::config::MLInferenceConfig; use crate::core::ml::drift_detector::DriftDetector; use crate::core::notification_service::NotificationService; use crate::core::playbook_service::PlaybookService; @@ -39,14 +38,15 @@ use crate::interface::port::notification::{AlertNotifier, NotificationConfigPort use crate::interface::port::repository::RepositoryPort; use crate::interface::port::secret_store::SecretStorePort; use crate::interface::port::soar::SoarPort; +use crate::model::access_control::list_type::ListType; use crate::model::detection::drift::FeatureBaselines; -use crate::model::monitoring::direction::FlowDirection; use crate::model::error::Error; use crate::model::error::ebpf::EbpfError; use crate::model::error::misc::MiscError; -use crate::model::access_control::list_type::ListType; use crate::model::log::ebpf::EbpfLog; use crate::model::log::system::SystemLog; +use crate::model::monitoring::direction::FlowDirection; +use crate::model::system::config::MLInferenceConfig; use macros::log; /// Holds all Arc-wrapped services that make up the running application. diff --git a/net-guardia/src/infrastructure/statistics.rs b/net-guardia/src/infrastructure/statistics.rs index c96248e..dd6299a 100644 --- a/net-guardia/src/infrastructure/statistics.rs +++ b/net-guardia/src/infrastructure/statistics.rs @@ -4,7 +4,9 @@ use std::time; use crate::core::ml::engine::Engine; use crate::core::ml::flow_tracker::FlowData; use crate::model::monitoring::direction::Direction; -use crate::model::monitoring::flow_stats::{FlowPushPayload, FlowStatsEntry, FlowSubscription, FlowSummary, StatsSummary}; +use crate::model::monitoring::flow_stats::{ + FlowPushPayload, FlowStatsEntry, FlowSubscription, FlowSummary, StatsSummary, +}; /// Conversion from core::ml::FlowData to model::FlowStatsEntry. /// Placed here (core layer) to maintain dependency rule: model/ must not import core/. diff --git a/net-guardia/src/interface/port/api_key.rs b/net-guardia/src/interface/port/api_key.rs index 9bedc15..9c5caa9 100644 --- a/net-guardia/src/interface/port/api_key.rs +++ b/net-guardia/src/interface/port/api_key.rs @@ -1,5 +1,5 @@ -use crate::model::identity::auth::Claims; use crate::model::error::Error; +use crate::model::identity::auth::Claims; /// Type alias for API key list items: (id, name, permission_level, created_at, last_used_at) #[allow(clippy::type_complexity)] diff --git a/net-guardia/src/model/detection/ml_detection.rs b/net-guardia/src/model/detection/ml_detection.rs index 76c1ef7..4c1f120 100644 --- a/net-guardia/src/model/detection/ml_detection.rs +++ b/net-guardia/src/model/detection/ml_detection.rs @@ -106,7 +106,6 @@ pub struct DetectionResult { pub ae_score: f32, pub anomaly_score: f32, pub c2_score: f32, - pub threshold: f32, pub packet_count: u64, pub flow_duration_us: u64, } diff --git a/net-guardia/src/model/system/config.rs b/net-guardia/src/model/system/config.rs index 70df18b..1c85f95 100644 --- a/net-guardia/src/model/system/config.rs +++ b/net-guardia/src/model/system/config.rs @@ -88,11 +88,17 @@ pub struct MLInferenceConfig { pub attack_labels: HashMap, pub anomaly_threshold: f32, pub c2_threshold: f32, + #[serde(default = "default_class_min_confidence")] + pub class_min_confidence: f32, pub model_type: String, pub output_names: Vec, pub ae_feature_weights: HashMap, } +fn default_class_min_confidence() -> f32 { + 0.4 +} + impl MLInferenceConfig { pub fn num_ae_features(&self) -> usize { self.ae_feature_names.len()