From 73ad4374bd3a492b1f17765cea87163a3579026f Mon Sep 17 00:00:00 2001 From: ParrotXray Date: Tue, 19 May 2026 07:54:59 +0000 Subject: [PATCH] feat: auto-select native ORT or ort-tract backend based on .so presence and ML+Rule fusion decision layer --- Cargo.lock | 13 +- net-guardia/Cargo.toml | 11 +- net-guardia/build.rs | 796 ++++++++++-------- net-guardia/src/core/ebpf/mod.rs | 12 +- net-guardia/src/core/ebpf/xsk_manager.rs | 24 +- .../src/core/infrastructure/ml_alert.rs | 39 - net-guardia/src/core/infrastructure/mod.rs | 31 +- net-guardia/src/core/system.rs | 10 +- net-guardia/src/detection/ml/engine.rs | 10 +- net-guardia/src/detection/ml/model_loader.rs | 16 +- net-guardia/src/detection/mod.rs | 1 + net-guardia/src/model/config.rs | 16 + net-guardia/src/model/log/ml.rs | 6 + net-guardia/src/model/ml_detection.rs | 97 ++- net-guardia/src/web/api/mod.rs | 4 +- .../src/web/websocket/alert_websocket.rs | 14 +- onnxruntime/info.txt | 2 + 17 files changed, 666 insertions(+), 436 deletions(-) delete mode 100644 net-guardia/src/core/infrastructure/ml_alert.rs create mode 100644 onnxruntime/info.txt diff --git a/Cargo.lock b/Cargo.lock index 2f99eaf..9a398ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -741,7 +741,7 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.8.9", ] [[package]] @@ -1707,6 +1707,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.15" @@ -2249,6 +2259,7 @@ version = "2.0.0-rc.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" dependencies = [ + "libloading 0.9.0", "ndarray 0.17.2", "ort-sys", "smallvec", diff --git a/net-guardia/Cargo.toml b/net-guardia/Cargo.toml index 01f5857..a420b1f 100644 --- a/net-guardia/Cargo.toml +++ b/net-guardia/Cargo.toml @@ -52,9 +52,14 @@ nom7 = { version = "7.1", package = "nom" } nom8 = { version = "8.0", package = "nom" } aes-gcm = "0.10" -ort-tract = "0.3.0+0.22" -ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "ndarray", "alternative-backend"] } +ort-tract = { version = "0.3.0+0.22", optional = true } +ort = { version = "=2.0.0-rc.12", default-features = false, features = ["std", "ndarray"] } ndarray = "0.17" + +[features] +default = ["native-ort-backend"] +tract-backend = ["dep:ort-tract", "ort/alternative-backend"] +native-ort-backend = ["ort/load-dynamic", "ort/api-18"] #csv = "1.4.0" #anyhow = "1.0.100" @@ -65,4 +70,4 @@ rusqlite = { version = "0.32", features = ["bundled"] } [[bin]] name = "net-guardia" -path = "src/main.rs" +path = "src/main.rs" \ No newline at end of file diff --git a/net-guardia/build.rs b/net-guardia/build.rs index 8dc1074..e9b23a7 100644 --- a/net-guardia/build.rs +++ b/net-guardia/build.rs @@ -27,40 +27,40 @@ const BUF_DNS_RCODE: u8 = 15; const BUF_DNS_QTYPE: u8 = 16; const BUF_DNS_ANSWER: u8 = 17; // SSH -const BUF_SSH_PROTO: u8 = 18; +const BUF_SSH_PROTO: u8 = 18; const BUF_SSH_SOFTWARE: u8 = 19; -const BUF_SSH_HASSH: u8 = 20; +const BUF_SSH_HASSH: u8 = 20; // SMTP -const BUF_SMTP_HELO: u8 = 21; -const BUF_SMTP_FROM: u8 = 22; -const BUF_SMTP_RCPT: u8 = 23; +const BUF_SMTP_HELO: u8 = 21; +const BUF_SMTP_FROM: u8 = 22; +const BUF_SMTP_RCPT: u8 = 23; const BUF_SMTP_REPLY: u8 = 24; // FTP -const BUF_FTP_COMMAND: u8 = 25; +const BUF_FTP_COMMAND: u8 = 25; const BUF_FTP_FILENAME: u8 = 26; -const BUF_FTP_REPLY: u8 = 27; +const BUF_FTP_REPLY: u8 = 27; // MQTT const BUF_MQTT_CLIENTID: u8 = 28; -const BUF_MQTT_TOPIC: u8 = 29; +const BUF_MQTT_TOPIC: u8 = 29; const BUF_MQTT_USERNAME: u8 = 30; // HTTP/2 -const BUF_HTTP2_METHOD: u8 = 31; -const BUF_HTTP2_PATH: u8 = 32; +const BUF_HTTP2_METHOD: u8 = 31; +const BUF_HTTP2_PATH: u8 = 32; const BUF_HTTP2_AUTHORITY: u8 = 33; -const BUF_HTTP2_STATUS: u8 = 34; -const BUF_HTTP2_UA: u8 = 35; +const BUF_HTTP2_STATUS: u8 = 34; +const BUF_HTTP2_UA: u8 = 35; // TLS extended -const BUF_TLS_JA3S: u8 = 36; -const BUF_TLS_JA4: u8 = 37; -const BUF_TLS_CERT_SUBJECT: u8 = 38; -const BUF_TLS_CERT_ISSUER: u8 = 39; +const BUF_TLS_JA3S: u8 = 36; +const BUF_TLS_JA4: u8 = 37; +const BUF_TLS_CERT_SUBJECT: u8 = 38; +const BUF_TLS_CERT_ISSUER: u8 = 39; const BUF_TLS_CERT_FINGERPRINT: u8 = 40; -const BUF_TLS_CERT_NOT_BEFORE: u8 = 41; -const BUF_TLS_CERT_NOT_AFTER: u8 = 42; -const BUF_TLS_ALPN: u8 = 43; +const BUF_TLS_CERT_NOT_BEFORE: u8 = 41; +const BUF_TLS_CERT_NOT_AFTER: u8 = 42; +const BUF_TLS_ALPN: u8 = 43; // QUIC -const BUF_QUIC_SNI: u8 = 44; -const BUF_QUIC_VERSION: u8 = 45; +const BUF_QUIC_SNI: u8 = 44; +const BUF_QUIC_VERSION: u8 = 45; struct ContentMatch { pattern: Vec, @@ -96,17 +96,17 @@ struct FlowbitEntry { /// Build-time mirror of rule_detection::ByteOp. Fields use i64 for SQLite. struct ByteOp { - kind: u8, // 0=byte_test 1=byte_jump 2=byte_extract 3=isdataat + kind: u8, // 0=byte_test 1=byte_jump 2=byte_extract 3=isdataat count: u8, offset: i32, relative: bool, endian_little: bool, string_mode: bool, - string_base: u8, // 0=dec 1=hex 2=oct + string_base: u8, // 0=dec 1=hex 2=oct buffer: u8, // byte_test - bt_op: u8, // 0=< 1=> 2=eq 3=ne 4=le 5=ge 6=band 7=bxor - bt_value: i64, // bit pattern stored as i64; reinterpreted as u64 at runtime + bt_op: u8, // 0=< 1=> 2=eq 3=ne 4=le 5=ge 6=band 7=bxor + bt_value: i64, // bit pattern stored as i64; reinterpreted as u64 at runtime bt_uses_var: bool, bt_bitmask: i64, bt_negated: bool, @@ -144,19 +144,61 @@ struct SigEntry { fast_pattern_idx: usize, pcres: Vec, flowbits: Vec, - alproto: u8, // 0 = any + alproto: u8, // 0 = any alproto_neg: bool, byte_ops: Vec, threshold: Option, } fn main() { + let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let db_dir = manifest_dir.join("static").join("db"); + let artifact_dir = manifest_dir.join("static").join("artifacts"); + let csv_dir = manifest_dir.parent().unwrap().join("records"); + let onnxruntime_dir = manifest_dir.parent().unwrap().join("onnxruntime").join("libonnxruntime.so"); + + let rules_dir = manifest_dir.join("static").join("rules"); + let static_web = manifest_dir.join("static").join("web"); + let project_name = manifest_dir.file_name().unwrap().to_string_lossy().into_owned(); + let frontend_dir = manifest_dir + .parent() + .unwrap() + .join(format!("{}-frontend", project_name)); + + // ── All cargo: directives in one place ──────────────────────────────── + + println!("cargo:rustc-env=RULES_DB_PATH={}", db_dir.display()); + println!("cargo:rustc-env=ARTIFACTCS_PATH={}", artifact_dir.display()); + println!("cargo:rustc-env=CSV_RECORD_PATH={}", csv_dir.display()); + println!("cargo:rustc-env=ONNXRUNTIME_PATH={}", onnxruntime_dir.display()); + + for item in &[ + "src", + "public", + "package.json", + "package-lock.json", + "next.config.js", + "tailwind.config.js", + "postcss.config.js", + "tsconfig.json", + ] { + println!("cargo:rerun-if-changed={}", frontend_dir.join(item).display()); + } + + println!("cargo:rerun-if-changed={}", rules_dir.display()); + println!("cargo:rerun-if-changed={}", rules_dir.join("suppress.conf").display()); + if let Ok(rd) = fs::read_dir(&rules_dir) { + for entry in rd.flatten() { + let p = entry.path(); + if p.extension().and_then(|e| e.to_str()) == Some("rules") { + println!("cargo:rerun-if-changed={}", p.display()); + } + } + } + + // ───────────────────────────────────────────────────────────────────── + if env::var_os("SKIP_EBPF_BUILD").is_some() { - // Emit placeholder env vars and stub eBPF binaries so the codebase - // type-checks without the eBPF / frontend toolchains installed. - println!("cargo:rustc-env=ARTIFACTCS_PATH="); - println!("cargo:rustc-env=CSV_RECORD_PATH="); - println!("cargo:rustc-env=RULES_DB_PATH="); let out = PathBuf::from(env::var_os("OUT_DIR").unwrap()); for name in &["net-guardia-ingress", "net-guardia-egress"] { let path = out.join(name); @@ -168,8 +210,8 @@ fn main() { } build_ingress_ebpf(); build_egress_ebpf(); - build_frontend(); - build_vectorscan_db(); + build_frontend(&frontend_dir, &static_web); + build_vectorscan_db(&db_dir, &rules_dir); } fn build_ingress_ebpf() { @@ -247,10 +289,10 @@ fn build_ingress_ebpf() { #[allow(clippy::collapsible_match)] match message.expect("valid JSON") { Message::CompilerArtifact(Artifact { - executable, - target: Target { name, .. }, - .. - }) => { + executable, + target: Target { name, .. }, + .. + }) => { if let Some(executable) = executable { executables.push((name, executable.into_std_path_buf())); } @@ -367,10 +409,10 @@ fn build_egress_ebpf() { #[allow(clippy::collapsible_match)] match message.expect("valid JSON") { Message::CompilerArtifact(Artifact { - executable, - target: Target { name, .. }, - .. - }) => { + executable, + target: Target { name, .. }, + .. + }) => { if let Some(executable) = executable { executables.push((name, executable.into_std_path_buf())); } @@ -411,72 +453,30 @@ fn build_egress_ebpf() { } } -fn build_frontend() { +fn build_frontend(frontend_dir: &PathBuf, static_dir: &PathBuf) { let _ = dotenvy::dotenv(); - // let Some(frontend_dir) = env::var_os("FRONTEND_DIR") else { - // panic!("FRONTEND_DIR environment variable is required but not set"); - // }; - - let project_root = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let static_dir = project_root.join("static").join("web"); - - let project_name = project_root.file_name().unwrap().to_string_lossy(); - let frontend_dir = project_root - .parent() - .unwrap() - .join(format!("{}-frontend", project_name)); - if !frontend_dir.exists() { panic!("Frontend directory {:?} does not exist", frontend_dir); } - println!("cargo:rerun-if-changed={}", frontend_dir.join("src").display()); - println!("cargo:rerun-if-changed={}", frontend_dir.join("public").display()); - println!("cargo:rerun-if-changed={}", frontend_dir.join("package.json").display()); - println!( - "cargo:rerun-if-changed={}", - frontend_dir.join("package-lock.json").display() - ); - println!( - "cargo:rerun-if-changed={}", - frontend_dir.join("next.config.js").display() - ); - println!( - "cargo:rerun-if-changed={}", - frontend_dir.join("tailwind.config.js").display() - ); - println!( - "cargo:rerun-if-changed={}", - frontend_dir.join("postcss.config.js").display() - ); - println!( - "cargo:rerun-if-changed={}", - frontend_dir.join("tsconfig.json").display() - ); - let out_dir = frontend_dir.join("out"); - let need_build = needs_frontend_rebuild(&frontend_dir, &out_dir, &static_dir); - if !need_build { + if !needs_frontend_rebuild(frontend_dir, &out_dir, static_dir) { return; } - let mut cmd = Command::new("npm"); - cmd.arg("install") - .current_dir(&frontend_dir); - - let status = cmd + let status = Command::new("npm") + .arg("install") + .current_dir(frontend_dir) .status() .unwrap_or_else(|err| panic!("failed to run npm install: {err}")); if !status.success() { panic!("npm install failed with exit code: {:?}", status.code()); } - let mut cmd = Command::new("npx"); - cmd.args(["next", "build"]) - .current_dir(&frontend_dir); - - let status = cmd + let status = Command::new("npx") + .args(["next", "build"]) + .current_dir(frontend_dir) .status() .unwrap_or_else(|err| panic!("failed to run next build: {err}")); if !status.success() { @@ -484,11 +484,10 @@ fn build_frontend() { } if static_dir.exists() { - fs::remove_dir_all(&static_dir).unwrap_or_else(|err| panic!("failed to remove {:?}: {err}", static_dir)); + fs::remove_dir_all(static_dir).unwrap_or_else(|err| panic!("failed to remove {:?}: {err}", static_dir)); } - fs::create_dir_all(&static_dir).unwrap_or_else(|err| panic!("failed to create {:?}: {err}", static_dir)); - - copy_dir_all(&out_dir, &static_dir).unwrap_or_else(|err| panic!("failed to copy frontend build: {err}")); + fs::create_dir_all(static_dir).unwrap_or_else(|err| panic!("failed to create {:?}: {err}", static_dir)); + copy_dir_all(&out_dir, static_dir).unwrap_or_else(|err| panic!("failed to copy frontend build: {err}")); } fn needs_frontend_rebuild(frontend_dir: &PathBuf, out_dir: &PathBuf, static_dir: &PathBuf) -> bool { @@ -589,38 +588,13 @@ fn copy_dir_all(src: &PathBuf, dst: &PathBuf) -> std::io::Result<()> { Ok(()) } +fn build_vectorscan_db(db_dir: &PathBuf, rules_dir: &PathBuf) { + let sigs = collect_sig_entries(rules_dir); + let suppress_rows = parse_suppress_conf(&rules_dir.join("suppress.conf")); -fn build_vectorscan_db() { - let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); - let out_dir = &manifest_dir.join("static").join("db"); - let artifact_dir = &manifest_dir.join("static").join("artifacts"); - let rules_dir = &manifest_dir.join("static").join("rules"); - - let csv_dir = &manifest_dir.parent().unwrap().join("records"); - - println!("cargo:rerun-if-changed={}", rules_dir.display()); - if let Ok(rd) = fs::read_dir(&rules_dir) { - for entry in rd.flatten() { - let p = entry.path(); - if p.extension().and_then(|e| e.to_str()) == Some("rules") { - println!("cargo:rerun-if-changed={}", p.display()); - } - } - } - - let sigs = collect_sig_entries(&rules_dir); - - let suppress_conf = rules_dir.join("suppress.conf"); - println!("cargo:rerun-if-changed={}", suppress_conf.display()); - let suppress_rows = parse_suppress_conf(&suppress_conf); - - let db_path = out_dir.join("rules.db"); + let db_path = db_dir.join("rules.db"); write_rules_db(&db_path, &sigs, &suppress_rows); - println!("cargo:rustc-env=RULES_DB_PATH={}", out_dir.display()); - println!("cargo:rustc-env=ARTIFACTCS_PATH={}", artifact_dir.display()); - println!("cargo:rustc-env=CSV_RECORD_PATH={}", csv_dir.display()); - let total_contents: usize = sigs.iter().map(|s| s.chain.len()).sum(); println!( "cargo:warning=NetGuardia: {} signatures ({} content entries) ready", @@ -696,58 +670,58 @@ fn collect_sig_entries(rules_dir: &PathBuf) -> Vec { /// Handles both new-style (`http.uri`) and old-style (`http_uri`) forms. fn detect_buffer_keyword(kw: &str) -> Option { match kw { - "http.uri" | "http_uri" => Some(BUF_HTTP_URI), - "http.header" | "http_header" => Some(BUF_HTTP_HEADER), - "http.request_body" | "http_client_body" => Some(BUF_HTTP_CLIENT_BODY), - "http.response_body"| "http_server_body" => Some(BUF_HTTP_SERVER_BODY), - "http.method" | "http_method" => Some(BUF_HTTP_METHOD), - "http.user_agent" | "http_user_agent" => Some(BUF_HTTP_USER_AGENT), - "http.host" | "http_host" => Some(BUF_HTTP_HOST), - "http.stat_code" | "http_stat_code" => Some(BUF_HTTP_STAT_CODE), - "http.cookie" | "http_cookie" => Some(BUF_HTTP_COOKIE), - "http.raw_uri" | "http_raw_uri" => Some(BUF_HTTP_RAW_URI), - "tls.sni" | "tls_sni" => Some(BUF_TLS_SNI), - "dns_query" | "dns.query" => Some(BUF_DNS_QUERY), - "tls.version" | "tls_version" => Some(BUF_TLS_VERSION), - "tls.ja3_hash" | "tls_ja3_hash" => Some(BUF_TLS_JA3), - "dns.rcode" | "dns_rcode" => Some(BUF_DNS_RCODE), - "dns.query.type" | "dns_query_type" => Some(BUF_DNS_QTYPE), - "dns.answer" | "dns_answer" => Some(BUF_DNS_ANSWER), + "http.uri" | "http_uri" => Some(BUF_HTTP_URI), + "http.header" | "http_header" => Some(BUF_HTTP_HEADER), + "http.request_body" | "http_client_body" => Some(BUF_HTTP_CLIENT_BODY), + "http.response_body" | "http_server_body" => Some(BUF_HTTP_SERVER_BODY), + "http.method" | "http_method" => Some(BUF_HTTP_METHOD), + "http.user_agent" | "http_user_agent" => Some(BUF_HTTP_USER_AGENT), + "http.host" | "http_host" => Some(BUF_HTTP_HOST), + "http.stat_code" | "http_stat_code" => Some(BUF_HTTP_STAT_CODE), + "http.cookie" | "http_cookie" => Some(BUF_HTTP_COOKIE), + "http.raw_uri" | "http_raw_uri" => Some(BUF_HTTP_RAW_URI), + "tls.sni" | "tls_sni" => Some(BUF_TLS_SNI), + "dns_query" | "dns.query" => Some(BUF_DNS_QUERY), + "tls.version" | "tls_version" => Some(BUF_TLS_VERSION), + "tls.ja3_hash" | "tls_ja3_hash" => Some(BUF_TLS_JA3), + "dns.rcode" | "dns_rcode" => Some(BUF_DNS_RCODE), + "dns.query.type" | "dns_query_type" => Some(BUF_DNS_QTYPE), + "dns.answer" | "dns_answer" => Some(BUF_DNS_ANSWER), // SSH - "ssh.proto" | "ssh_proto" => Some(BUF_SSH_PROTO), - "ssh.software" | "ssh_software" => Some(BUF_SSH_SOFTWARE), - "ssh.hassh" | "ssh_hassh" => Some(BUF_SSH_HASSH), + "ssh.proto" | "ssh_proto" => Some(BUF_SSH_PROTO), + "ssh.software" | "ssh_software" => Some(BUF_SSH_SOFTWARE), + "ssh.hassh" | "ssh_hassh" => Some(BUF_SSH_HASSH), // SMTP - "smtp.helo" | "smtp_helo" => Some(BUF_SMTP_HELO), - "smtp.mail_from" | "smtp_from" => Some(BUF_SMTP_FROM), - "smtp.rcpt_to" | "smtp_rcpt" => Some(BUF_SMTP_RCPT), - "smtp.reply_code" | "smtp_reply" => Some(BUF_SMTP_REPLY), + "smtp.helo" | "smtp_helo" => Some(BUF_SMTP_HELO), + "smtp.mail_from" | "smtp_from" => Some(BUF_SMTP_FROM), + "smtp.rcpt_to" | "smtp_rcpt" => Some(BUF_SMTP_RCPT), + "smtp.reply_code" | "smtp_reply" => Some(BUF_SMTP_REPLY), // FTP - "ftp.command" | "ftp_command" => Some(BUF_FTP_COMMAND), - "ftp.filename" | "ftp_filename" => Some(BUF_FTP_FILENAME), - "ftp.reply_code" | "ftp_reply" => Some(BUF_FTP_REPLY), + "ftp.command" | "ftp_command" => Some(BUF_FTP_COMMAND), + "ftp.filename" | "ftp_filename" => Some(BUF_FTP_FILENAME), + "ftp.reply_code" | "ftp_reply" => Some(BUF_FTP_REPLY), // MQTT - "mqtt.client_id" | "mqtt_client_id" => Some(BUF_MQTT_CLIENTID), - "mqtt.topic" | "mqtt_topic" => Some(BUF_MQTT_TOPIC), - "mqtt.username" | "mqtt_username" => Some(BUF_MQTT_USERNAME), + "mqtt.client_id" | "mqtt_client_id" => Some(BUF_MQTT_CLIENTID), + "mqtt.topic" | "mqtt_topic" => Some(BUF_MQTT_TOPIC), + "mqtt.username" | "mqtt_username" => Some(BUF_MQTT_USERNAME), // HTTP/2 - "http2.method" | "http2_method" => Some(BUF_HTTP2_METHOD), - "http2.path" | "http2_path" => Some(BUF_HTTP2_PATH), - "http2.authority" | "http2_authority" => Some(BUF_HTTP2_AUTHORITY), - "http2.status_code" | "http2_status" => Some(BUF_HTTP2_STATUS), - "http2.user_agent" | "http2_user_agent" => Some(BUF_HTTP2_UA), + "http2.method" | "http2_method" => Some(BUF_HTTP2_METHOD), + "http2.path" | "http2_path" => Some(BUF_HTTP2_PATH), + "http2.authority" | "http2_authority" => Some(BUF_HTTP2_AUTHORITY), + "http2.status_code" | "http2_status" => Some(BUF_HTTP2_STATUS), + "http2.user_agent" | "http2_user_agent" => Some(BUF_HTTP2_UA), // TLS extended - "tls.ja3s_hash" | "tls_ja3s_hash" => Some(BUF_TLS_JA3S), - "tls.ja4_hash" | "tls_ja4_hash" => Some(BUF_TLS_JA4), - "tls.cert_subject" | "tls_cert_subject" => Some(BUF_TLS_CERT_SUBJECT), - "tls.cert_issuer" | "tls_cert_issuer" => Some(BUF_TLS_CERT_ISSUER), - "tls.fingerprint" | "tls_fingerprint" => Some(BUF_TLS_CERT_FINGERPRINT), - "tls.notbefore" | "tls_notbefore" => Some(BUF_TLS_CERT_NOT_BEFORE), - "tls.notafter" | "tls_notafter" => Some(BUF_TLS_CERT_NOT_AFTER), - "tls.alpn" | "tls_alpn" => Some(BUF_TLS_ALPN), + "tls.ja3s_hash" | "tls_ja3s_hash" => Some(BUF_TLS_JA3S), + "tls.ja4_hash" | "tls_ja4_hash" => Some(BUF_TLS_JA4), + "tls.cert_subject" | "tls_cert_subject" => Some(BUF_TLS_CERT_SUBJECT), + "tls.cert_issuer" | "tls_cert_issuer" => Some(BUF_TLS_CERT_ISSUER), + "tls.fingerprint" | "tls_fingerprint" => Some(BUF_TLS_CERT_FINGERPRINT), + "tls.notbefore" | "tls_notbefore" => Some(BUF_TLS_CERT_NOT_BEFORE), + "tls.notafter" | "tls_notafter" => Some(BUF_TLS_CERT_NOT_AFTER), + "tls.alpn" | "tls_alpn" => Some(BUF_TLS_ALPN), // QUIC - "quic.sni" | "quic_sni" => Some(BUF_QUIC_SNI), - "quic.version" | "quic_version" => Some(BUF_QUIC_VERSION), + "quic.sni" | "quic_sni" => Some(BUF_QUIC_SNI), + "quic.version" | "quic_version" => Some(BUF_QUIC_VERSION), _ => None, } } @@ -756,7 +730,7 @@ fn detect_buffer_keyword(kw: &str) -> Option { /// splitting on `;` while respecting double-quoted strings. fn tokenize_options(rule: &str) -> Vec { let opts_start = rule.find('(').map(|p| p + 1).unwrap_or(0); - let opts_end = rule.rfind(')').unwrap_or(rule.len()); + let opts_end = rule.rfind(')').unwrap_or(rule.len()); let opts = &rule[opts_start..opts_end.max(opts_start)]; let mut tokens: Vec = Vec::new(); @@ -865,7 +839,9 @@ fn extract_content_chain(rule: &str) -> Vec { } i += 1; // Stop if new-style (becomes sticky for next content). - if mod_tok.contains('.') { break; } + if mod_tok.contains('.') { + break; + } continue; } @@ -939,7 +915,8 @@ fn select_fast_pattern(chain: &[ContentMatch]) -> Option { if let Some(idx) = chain.iter().position(|e| e.is_fast_pattern && !e.negated) { return Some(idx); } - chain.iter() + chain + .iter() .enumerate() .filter(|(_, e)| !e.negated && e.pattern.len() >= 4) .max_by_key(|(_, e)| e.pattern.len()) @@ -970,8 +947,12 @@ fn parse_suppress_conf(path: &Path) -> Vec { let mut rows = Vec::new(); for line in text.lines() { let line = line.trim(); - if line.starts_with('#') || line.is_empty() { continue; } - if !line.starts_with("suppress") { continue; } + if line.starts_with('#') || line.is_empty() { + continue; + } + if !line.starts_with("suppress") { + continue; + } let mut sid = 0u32; let mut track = 0u8; let mut ip_net = String::new(); @@ -989,7 +970,9 @@ fn parse_suppress_conf(path: &Path) -> Vec { ip_net = v.trim().to_string(); } } - if sid == 0 { continue; } + if sid == 0 { + continue; + } rows.push(SuppressRow { sid, track, ip_net }); } rows @@ -998,10 +981,10 @@ fn parse_suppress_conf(path: &Path) -> Vec { fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { let _ = fs::remove_file(path); - let conn = rusqlite::Connection::open(path) - .unwrap_or_else(|e| panic!("failed to open rules.db at {path:?}: {e}")); + let conn = rusqlite::Connection::open(path).unwrap_or_else(|e| panic!("failed to open rules.db at {path:?}: {e}")); - conn.execute_batch(" + conn.execute_batch( + " CREATE TABLE patterns ( id INTEGER PRIMARY KEY, expression BLOB NOT NULL, @@ -1089,9 +1072,12 @@ fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { count INTEGER NOT NULL, seconds INTEGER NOT NULL ); - ").unwrap_or_else(|e| panic!("rules.db schema failed: {e}")); + ", + ) + .unwrap_or_else(|e| panic!("rules.db schema failed: {e}")); - let tx = conn.unchecked_transaction() + let tx = conn + .unchecked_transaction() .unwrap_or_else(|e| panic!("rules.db transaction failed: {e}")); for (sig_idx, sig) in sigs.iter().enumerate() { @@ -1102,7 +1088,8 @@ fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { tx.execute( "INSERT INTO patterns (id, expression, nocase) VALUES (?1, ?2, ?3)", rusqlite::params![id, expression, fp.nocase as i64], - ).unwrap(); + ) + .unwrap(); tx.execute( "INSERT INTO signatures (id, sid, flow_dir, flow_established, src_ip_flag, dst_ip_flag, msg, alproto, alproto_neg) \ @@ -1124,7 +1111,8 @@ fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { tx.execute( "INSERT INTO signature_ports (sig_id, dport) VALUES (?1, ?2)", rusqlite::params![id, dport as i64], - ).unwrap(); + ) + .unwrap(); } for (order, entry) in sig.chain.iter().enumerate() { @@ -1150,7 +1138,8 @@ fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { entry.depth as i64, entry.buffer as i64, ], - ).unwrap(); + ) + .unwrap(); } for (order, entry) in sig.pcres.iter().enumerate() { @@ -1164,14 +1153,16 @@ fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { entry.negated as i64, entry.buffer as i64, ], - ).unwrap(); + ) + .unwrap(); } for entry in &sig.flowbits { tx.execute( "INSERT INTO flowbit_entries (sig_id, op, name) VALUES (?1, ?2, ?3)", rusqlite::params![id, entry.op as i64, &entry.name], - ).unwrap(); + ) + .unwrap(); } if let Some(t) = &sig.threshold { @@ -1179,7 +1170,8 @@ fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { "INSERT INTO threshold_entries (sig_id, kind, track, count, seconds) \ VALUES (?1, ?2, ?3, ?4, ?5)", rusqlite::params![id, t.kind as i64, t.track as i64, t.count as i64, t.seconds as i64], - ).unwrap(); + ) + .unwrap(); } for (order, op) in sig.byte_ops.iter().enumerate() { @@ -1213,21 +1205,25 @@ fn write_rules_db(path: &Path, sigs: &[SigEntry], suppress: &[SuppressRow]) { op.be_var_idx as i64, op.be_multiplier, ], - ).unwrap(); + ) + .unwrap(); } } tx.commit().unwrap_or_else(|e| panic!("rules.db commit failed: {e}")); - let tx2 = conn.unchecked_transaction() + let tx2 = conn + .unchecked_transaction() .unwrap_or_else(|e| panic!("rules.db suppress transaction failed: {e}")); for (i, row) in suppress.iter().enumerate() { tx2.execute( "INSERT INTO suppress (id, sid, track, ip_net) VALUES (?1, ?2, ?3, ?4)", rusqlite::params![i as i64, row.sid as i64, row.track as i64, &row.ip_net], - ).unwrap(); + ) + .unwrap(); } - tx2.commit().unwrap_or_else(|e| panic!("rules.db suppress commit failed: {e}")); + tx2.commit() + .unwrap_or_else(|e| panic!("rules.db suppress commit failed: {e}")); } fn extract_sid(rule: &str) -> Option { @@ -1300,8 +1296,7 @@ fn regex_escape(bytes: &[u8]) -> Vec { for &b in bytes { if matches!( b, - b'.' | b'^' | b'$' | b'*' | b'+' | b'?' | b'(' | b')' | - b'[' | b']' | b'{' | b'}' | b'\\' | b'|' + b'.' | b'^' | b'$' | b'*' | b'+' | b'?' | b'(' | b')' | b'[' | b']' | b'{' | b'}' | b'\\' | b'|' ) { out.push(b'\\'); } @@ -1316,9 +1311,13 @@ fn extract_flow_dir(rule: &str) -> u8 { let rest = &rule[pos + 5..]; let seg_end = rest.find(';').unwrap_or(rest.len()); let opts = &rest[..seg_end]; - if opts.contains("to_server") { 1 } - else if opts.contains("to_client") { 2 } - else { 0 } + if opts.contains("to_server") { + 1 + } else if opts.contains("to_client") { + 2 + } else { + 0 + } } /// Parse destination ports from the rule header (token index 6, before the '('). @@ -1345,57 +1344,57 @@ fn extract_dst_ports(rule: &str) -> Vec { fn expand_port_token(token: &str) -> Vec { match token { // HTTP / Web - "$HTTP_PORTS" => return vec![80, 443, 8080, 8443, 8000, 8888], - "$HTTP_PORTS2" => return vec![80, 8080], - "$HTTPS_PORTS" => return vec![443, 8443], - "$FILE_DATA_PORTS" => return vec![80, 443, 8080, 8443, 110, 143], - "$PROXY_PORTS" => return vec![3128, 8080, 8118, 8888], + "$HTTP_PORTS" => return vec![80, 443, 8080, 8443, 8000, 8888], + "$HTTP_PORTS2" => return vec![80, 8080], + "$HTTPS_PORTS" => return vec![443, 8443], + "$FILE_DATA_PORTS" => return vec![80, 443, 8080, 8443, 110, 143], + "$PROXY_PORTS" => return vec![3128, 8080, 8118, 8888], // Mail - "$SMTP_PORTS" => return vec![25, 587, 465], - "$IMAP_PORTS" => return vec![143, 993], - "$POP3_PORTS" => return vec![110, 995], - // File transfer / Remote - "$FTP_PORTS" => return vec![21], - "$FTP_DATA" => return vec![20], - "$SSH_PORTS" => return vec![22], - "$TELNET_PORTS" => return vec![23], - "$RDP_PORTS" => return vec![3389], - "$VNC_PORTS" => return vec![5900, 5901, 5902, 5903], + "$SMTP_PORTS" => return vec![25, 587, 465], + "$IMAP_PORTS" => return vec![143, 993], + "$POP3_PORTS" => return vec![110, 995], + // File transfer / Remote + "$FTP_PORTS" => return vec![21], + "$FTP_DATA" => return vec![20], + "$SSH_PORTS" => return vec![22], + "$TELNET_PORTS" => return vec![23], + "$RDP_PORTS" => return vec![3389], + "$VNC_PORTS" => return vec![5900, 5901, 5902, 5903], // Database - "$SQL_PORTS" => return vec![3306, 5432, 1433, 1521], - "$ORACLE_PORTS" => return vec![1521, 1526], - "$MSSQL_PORTS" => return vec![1433, 1434], - "$MYSQL_PORTS" => return vec![3306], - "$PGSQL_PORTS" => return vec![5432], - "$MONGODB_PORTS" => return vec![27017, 27018], - "$REDIS_PORTS" => return vec![6379], - "$MEMCACHED_PORTS" => return vec![11211], + "$SQL_PORTS" => return vec![3306, 5432, 1433, 1521], + "$ORACLE_PORTS" => return vec![1521, 1526], + "$MSSQL_PORTS" => return vec![1433, 1434], + "$MYSQL_PORTS" => return vec![3306], + "$PGSQL_PORTS" => return vec![5432], + "$MONGODB_PORTS" => return vec![27017, 27018], + "$REDIS_PORTS" => return vec![6379], + "$MEMCACHED_PORTS" => return vec![11211], "$ELASTICSEARCH_PORTS" => return vec![9200, 9300], // DNS / Directory - "$DNS_PORTS" => return vec![53], - "$LDAP_PORTS" => return vec![389, 636, 3268, 3269], - "$KERBEROS_PORTS" => return vec![88, 464], + "$DNS_PORTS" => return vec![53], + "$LDAP_PORTS" => return vec![389, 636, 3268, 3269], + "$KERBEROS_PORTS" => return vec![88, 464], // VoIP / Messaging - "$SIP_PORTS" => return vec![5060, 5061], - "$IRC_PORTS" => return vec![6667, 6668, 6669, 7000], - "$JABBER_PORTS" => return vec![5222, 5223], - "$MSN_PORTS" => return vec![1863], - "$AIM_PORTS" => return vec![5190], + "$SIP_PORTS" => return vec![5060, 5061], + "$IRC_PORTS" => return vec![6667, 6668, 6669, 7000], + "$JABBER_PORTS" => return vec![5222, 5223], + "$MSN_PORTS" => return vec![1863], + "$AIM_PORTS" => return vec![5190], // Network management - "$SNMP_PORTS" => return vec![161, 162], - "$SYSLOG_PORTS" => return vec![514], + "$SNMP_PORTS" => return vec![161, 162], + "$SYSLOG_PORTS" => return vec![514], // Industrial / SCADA - "$MODBUS_PORTS" => return vec![502], - "$DNP3_PORTS" => return vec![20000], - "$ENIP_PORTS" => return vec![44818], - "$BACnet_PORTS" => return vec![47808], - "$VXLAN_PORTS" => return vec![4789], - "$TEREDO_PORTS" => return vec![3544], + "$MODBUS_PORTS" => return vec![502], + "$DNP3_PORTS" => return vec![20000], + "$ENIP_PORTS" => return vec![44818], + "$BACnet_PORTS" => return vec![47808], + "$VXLAN_PORTS" => return vec![4789], + "$TEREDO_PORTS" => return vec![3544], // Shellcode / generic catch-alls // Suricata default: !80 — we can't enumerate the complement, treat as any - "$SHELLCODE_PORTS" => return Vec::new(), + "$SHELLCODE_PORTS" => return Vec::new(), // Wildcard - "any" | "!any" => return Vec::new(), + "any" | "!any" => return Vec::new(), _ => {} } @@ -1423,7 +1422,9 @@ fn expand_port_token(token: &str) -> Vec { let mut has_negation_only = true; for part in inner.split(',') { let part = part.trim(); - if part.is_empty() { continue; } + if part.is_empty() { + continue; + } if part.starts_with('!') { // negated entry — skip but don't block the group continue; @@ -1442,7 +1443,9 @@ fn expand_port_token(token: &str) -> Vec { if hi.saturating_sub(lo) > 1024 { return Vec::new(); } - for p in lo..=hi { ports.push(p); } + for p in lo..=hi { + ports.push(p); + } continue; } if let Ok(p) = part.parse::() { @@ -1484,10 +1487,10 @@ fn ip_token_to_flag(token: &str) -> u8 { token }; match t { - "$HOME_NET" => 1, - "!$HOME_NET" => 2, - "$EXTERNAL_NET" => 2, - "!$EXTERNAL_NET" => 1, + "$HOME_NET" => 1, + "!$HOME_NET" => 2, + "$EXTERNAL_NET" => 2, + "!$EXTERNAL_NET" => 1, _ => 0, } } @@ -1502,14 +1505,22 @@ fn extract_pcre_entries(rule: &str) -> Vec { let mut entries = Vec::new(); for tok in &tokens { - let rest = if let Some(r) = tok.strip_prefix("pcre:") { r } else { continue }; + let rest = if let Some(r) = tok.strip_prefix("pcre:") { + r + } else { + continue; + }; let negated = rest.starts_with('!'); let quoted = if negated { &rest[1..] } else { rest }; - if !quoted.starts_with('"') { continue; } + if !quoted.starts_with('"') { + continue; + } let inner = unquote_content(quoted); - if !inner.starts_with('/') { continue; } + if !inner.starts_with('/') { + continue; + } let body = &inner[1..]; let Some(close) = body.rfind('/') else { continue }; @@ -1526,14 +1537,14 @@ fn extract_pcre_entries(rule: &str) -> Vec { 'm' => regex_mods.push('m'), 'U' | 'I' => buffer = BUF_HTTP_URI, 'H' | 'D' => buffer = BUF_HTTP_HEADER, - 'P' => buffer = BUF_HTTP_CLIENT_BODY, - 'Q' => buffer = BUF_HTTP_SERVER_BODY, - 'M' => buffer = BUF_HTTP_METHOD, - 'C' => buffer = BUF_HTTP_COOKIE, - 'S' => buffer = BUF_HTTP_STAT_CODE, - 'V' => buffer = BUF_HTTP_USER_AGENT, - 'W' => buffer = BUF_HTTP_HOST, - _ => {} + 'P' => buffer = BUF_HTTP_CLIENT_BODY, + 'Q' => buffer = BUF_HTTP_SERVER_BODY, + 'M' => buffer = BUF_HTTP_METHOD, + 'C' => buffer = BUF_HTTP_COOKIE, + 'S' => buffer = BUF_HTTP_STAT_CODE, + 'V' => buffer = BUF_HTTP_USER_AGENT, + 'W' => buffer = BUF_HTTP_HOST, + _ => {} } } @@ -1543,7 +1554,11 @@ fn extract_pcre_entries(rule: &str) -> Vec { format!("(?{}){}", regex_mods, pattern_str) }; - entries.push(PcreEntry { pattern: full_pattern, negated, buffer }); + entries.push(PcreEntry { + pattern: full_pattern, + negated, + buffer, + }); } entries @@ -1560,18 +1575,22 @@ fn extract_alproto(rule: &str) -> (u8, bool) { let rest = &rule[pos..]; let end = rest.find(';').unwrap_or(rest.len()); let val = rest[..end].trim(); - let (negated, name) = if val.starts_with('!') { (true, val[1..].trim()) } else { (false, val) }; + let (negated, name) = if val.starts_with('!') { + (true, val[1..].trim()) + } else { + (false, val) + }; let id = match name { "http" | "http-any" => 1u8, - "http2" => 2, - "tls" | "ssl" => 3, - "dns" => 4, - "ssh" => 5, - "smtp" => 6, - "ftp" => 7, - "mqtt" => 8, - "quic" => 9, - _ => 0, + "http2" => 2, + "tls" | "ssl" => 3, + "dns" => 4, + "ssh" => 5, + "smtp" => 6, + "ftp" => 7, + "mqtt" => 8, + "quic" => 9, + _ => 0, }; (id, negated) } @@ -1596,19 +1615,19 @@ fn extract_threshold(rule: &str) -> Option { let part = part.trim(); if let Some(v) = part.strip_prefix("type ") { kind = match v.trim() { - "limit" => 1, - "both" => 2, + "limit" => 1, + "both" => 2, "threshold" => 3, - _ => 0, + _ => 0, }; } else if let Some(v) = part.strip_prefix("track ") { track = match v.trim() { - "by_dst" => 1, - "by_src" => 2, + "by_dst" => 1, + "by_src" => 2, "by_rule" => 3, "by_both" => 5, "by_flow" => 6, - _ => 3, + _ => 3, }; } else if let Some(v) = part.strip_prefix("count ") { count = v.trim().parse().unwrap_or(0); @@ -1621,7 +1640,12 @@ fn extract_threshold(rule: &str) -> Option { return None; } - Some(ThresholdEntry { kind, track, count, seconds }) + Some(ThresholdEntry { + kind, + track, + count, + seconds, + }) } /// Parse all byte_test / byte_jump / byte_extract / isdataat keywords from a rule. @@ -1678,25 +1702,24 @@ fn parse_byte_test(args: &str, var_names: &[String], buffer: u8) -> Option 0, - ">" => 1, + "<" => 0, + ">" => 1, "=" | "==" => 2, - "!=" => 3, - "<=" => 4, - ">=" => 5, - "&" => 6, - "^" => 7, - _ => return None, + "!=" => 3, + "<=" => 4, + ">=" => 5, + "&" => 6, + "^" => 7, + _ => return None, }; let val_str = parts[2].trim(); - let (bt_value, bt_uses_var) = - if let Some(idx) = var_names.iter().position(|n| n == val_str) { - (idx as i64, true) - } else { - let n = parse_num_u64(val_str).unwrap_or(0); - (n as i64, false) - }; + let (bt_value, bt_uses_var) = if let Some(idx) = var_names.iter().position(|n| n == val_str) { + (idx as i64, true) + } else { + let n = parse_num_u64(val_str).unwrap_or(0); + (n as i64, false) + }; let offset = parts[3].trim().parse::().unwrap_or(0); @@ -1709,14 +1732,23 @@ fn parse_byte_test(args: &str, var_names: &[String], buffer: u8) -> Option relative = true, - "little" => endian_little = true, - "big" => endian_little = false, - "string" => string_mode = true, - "hex" => { string_mode = true; string_base = 1; } - "dec" => { string_mode = true; string_base = 0; } - "oct" => { string_mode = true; string_base = 2; } - "bitmask" => { + "relative" => relative = true, + "little" => endian_little = true, + "big" => endian_little = false, + "string" => string_mode = true, + "hex" => { + string_mode = true; + string_base = 1; + } + "dec" => { + string_mode = true; + string_base = 0; + } + "oct" => { + string_mode = true; + string_base = 2; + } + "bitmask" => { i += 1; if let Some(mask_str) = parts.get(i) { bt_bitmask = parse_num_u64(mask_str.trim()).unwrap_or(0) as i64; @@ -1728,11 +1760,25 @@ fn parse_byte_test(args: &str, var_names: &[String], buffer: u8) -> Option Option { let mut i = 2; while i < parts.len() { match parts[i] { - "relative" => relative = true, - "little" => endian_little = true, - "big" => endian_little = false, - "string" => string_mode = true, - "hex" => { string_mode = true; string_base = 1; } - "dec" => { string_mode = true; string_base = 0; } - "oct" => { string_mode = true; string_base = 2; } - "align" => bj_align = true, + "relative" => relative = true, + "little" => endian_little = true, + "big" => endian_little = false, + "string" => string_mode = true, + "hex" => { + string_mode = true; + string_base = 1; + } + "dec" => { + string_mode = true; + string_base = 0; + } + "oct" => { + string_mode = true; + string_base = 2; + } + "align" => bj_align = true, "from_beginning" => bj_from_beginning = true, - "multiplier" => { + "multiplier" => { i += 1; if let Some(v) = parts.get(i) { bj_multiplier = v.trim().parse::().unwrap_or(1); } } - "post_offset" => { + "post_offset" => { i += 1; if let Some(v) = parts.get(i) { bj_post_offset = v.trim().parse::().unwrap_or(0); @@ -1785,11 +1840,25 @@ fn parse_byte_jump(args: &str, buffer: u8) -> Option { } Some(ByteOp { - kind: 1, count, offset, relative, endian_little, - string_mode, string_base, buffer, - bt_op: 0, bt_value: 0, bt_uses_var: false, bt_bitmask: 0, bt_negated: false, - bj_align, bj_multiplier, bj_from_beginning, bj_post_offset, - be_var_idx: 0, be_multiplier: 1, + kind: 1, + count, + offset, + relative, + endian_little, + string_mode, + string_base, + buffer, + bt_op: 0, + bt_value: 0, + bt_uses_var: false, + bt_bitmask: 0, + bt_negated: false, + bj_align, + bj_multiplier, + bj_from_beginning, + bj_post_offset, + be_var_idx: 0, + be_multiplier: 1, }) } @@ -1805,8 +1874,7 @@ fn parse_byte_extract(args: &str, var_names: &[String], buffer: u8) -> Option<(B let offset = parts[1].parse::().unwrap_or(0); let name = parts[2].trim().to_string(); - let be_var_idx = var_names.iter().position(|n| n == &name) - .unwrap_or(var_names.len()) as u8; + let be_var_idx = var_names.iter().position(|n| n == &name).unwrap_or(var_names.len()) as u8; let mut relative = false; let mut endian_little = false; @@ -1817,13 +1885,22 @@ fn parse_byte_extract(args: &str, var_names: &[String], buffer: u8) -> Option<(B let mut i = 3; while i < parts.len() { match parts[i] { - "relative" => relative = true, - "little" => endian_little = true, - "big" => endian_little = false, - "string" => string_mode = true, - "hex" => { string_mode = true; string_base = 1; } - "dec" => { string_mode = true; string_base = 0; } - "oct" => { string_mode = true; string_base = 2; } + "relative" => relative = true, + "little" => endian_little = true, + "big" => endian_little = false, + "string" => string_mode = true, + "hex" => { + string_mode = true; + string_base = 1; + } + "dec" => { + string_mode = true; + string_base = 0; + } + "oct" => { + string_mode = true; + string_base = 2; + } "multiplier" => { i += 1; if let Some(v) = parts.get(i) { @@ -1835,13 +1912,30 @@ fn parse_byte_extract(args: &str, var_names: &[String], buffer: u8) -> Option<(B i += 1; } - Some((ByteOp { - kind: 2, count, offset, relative, endian_little, - string_mode, string_base, buffer, - bt_op: 0, bt_value: 0, bt_uses_var: false, bt_bitmask: 0, bt_negated: false, - bj_align: false, bj_multiplier: 1, bj_from_beginning: false, bj_post_offset: 0, - be_var_idx, be_multiplier, - }, name)) + Some(( + ByteOp { + kind: 2, + count, + offset, + relative, + endian_little, + string_mode, + string_base, + buffer, + bt_op: 0, + bt_value: 0, + bt_uses_var: false, + bt_bitmask: 0, + bt_negated: false, + bj_align: false, + bj_multiplier: 1, + bj_from_beginning: false, + bj_post_offset: 0, + be_var_idx, + be_multiplier, + }, + name, + )) } /// Parse an `isdataat:[!]offset[,relative]` argument string. @@ -1868,11 +1962,25 @@ fn parse_isdataat(args: &str, buffer: u8) -> Option { } Some(ByteOp { - kind: 3, count: 0, offset, relative, endian_little: false, - string_mode: false, string_base: 0, buffer, - bt_op: 0, bt_value: 0, bt_uses_var: false, bt_bitmask: 0, bt_negated, - bj_align: false, bj_multiplier: 1, bj_from_beginning: false, bj_post_offset: 0, - be_var_idx: 0, be_multiplier: 1, + kind: 3, + count: 0, + offset, + relative, + endian_little: false, + string_mode: false, + string_base: 0, + buffer, + bt_op: 0, + bt_value: 0, + bt_uses_var: false, + bt_bitmask: 0, + bt_negated, + bj_align: false, + bj_multiplier: 1, + bj_from_beginning: false, + bj_post_offset: 0, + be_var_idx: 0, + be_multiplier: 1, }) } @@ -1894,19 +2002,23 @@ fn extract_flowbit_entries(rule: &str) -> Vec { let mut entries = Vec::new(); for tok in &tokens { - let rest = if let Some(r) = tok.strip_prefix("flowbits:") { r } else { continue }; + let rest = if let Some(r) = tok.strip_prefix("flowbits:") { + r + } else { + continue; + }; let mut parts = rest.splitn(2, ','); let op_str = parts.next().unwrap_or("").trim(); let name = parts.next().unwrap_or("").trim().to_string(); let op = match op_str { - "set" => 0u8, - "toggle" => 1, - "unset" => 2, + "set" => 0u8, + "toggle" => 1, + "unset" => 2, "isnotset" => 3, - "isset" => 4, - "noalert" => 5, + "isset" => 4, + "noalert" => 5, _ => continue, }; @@ -1914,4 +2026,4 @@ fn extract_flowbit_entries(rule: &str) -> Vec { } entries -} \ No newline at end of file +} diff --git a/net-guardia/src/core/ebpf/mod.rs b/net-guardia/src/core/ebpf/mod.rs index 27f1a87..ff32e3d 100644 --- a/net-guardia/src/core/ebpf/mod.rs +++ b/net-guardia/src/core/ebpf/mod.rs @@ -15,9 +15,10 @@ use crate::core::ebpf::service::Service; use crate::core::ebpf::statistics::Statistics; use crate::core::ebpf::xsk_manager::XskManager; use crate::core::infrastructure::app_config::AppConfig; +use crate::detection::fusion::FusionEngine; +use crate::detection::ml::engine::Engine; use crate::model::error::system::SystemError; use crate::model::error::Error; -use crate::detection::ml::engine::Engine; pub struct EbpfServices { pub xsk_manager: Arc, @@ -47,14 +48,17 @@ impl EbpfServices { Ok(ebpf_services) } - pub async fn run(self: Arc, ml_engine: Arc) -> Result<(), Error> { + pub async fn run( + self: Arc, + ml_engine: Arc, + fusion_engine: Arc, + ) -> Result<(), Error> { let xsk_manager = self.xsk_manager.clone(); let statistics = self.statistics.clone(); - xsk_manager.run(Some(ml_engine), &self.shutdowns)?; + xsk_manager.run(Some(ml_engine), Some(fusion_engine), &self.shutdowns)?; let statistics_shutdown = statistics.run().await; - self.shutdowns.push(statistics_shutdown); Ok(()) } diff --git a/net-guardia/src/core/ebpf/xsk_manager.rs b/net-guardia/src/core/ebpf/xsk_manager.rs index 5c7bed3..6f23cd6 100644 --- a/net-guardia/src/core/ebpf/xsk_manager.rs +++ b/net-guardia/src/core/ebpf/xsk_manager.rs @@ -17,6 +17,7 @@ use xsk_rs::config::{BindFlags, FrameSize, Interface, LibxdpFlags, QueueSize, So use xsk_rs::{CompQueue, FillQueue, FrameDesc, RxQueue, Socket, TxQueue, Umem}; use crate::core::infrastructure::app_config::AppConfig; +use crate::detection::fusion::FusionEngine; use crate::detection::ml::engine::Engine; use crate::detection::rule::rule_engine::RuleEngine; use crate::detection::rule::stream_reassembler::StreamReassembler; @@ -53,7 +54,12 @@ impl XskManager { }) } - pub fn run(&self, engine: Option>, shutdowns: &SegQueue>) -> Result<(), Error> { + pub fn run( + &self, + engine: Option>, + fusion_engine: Option>, + shutdowns: &SegQueue>, + ) -> Result<(), Error> { let config = self.app_config.config.clone(); let combined_queue_count = config.combined_queue_count; @@ -82,6 +88,7 @@ impl XskManager { Direction::Ingress, engine.clone(), rule_engine.clone(), + fusion_engine.clone(), min_sig, )?; @@ -93,6 +100,7 @@ impl XskManager { Direction::Egress, engine.clone(), rule_engine.clone(), + fusion_engine.clone(), min_sig, )?; @@ -135,6 +143,7 @@ pub struct XskPair { frame_pool: Arc>>, engine: Option>, rule_engine: Option>, + fusion_engine: Option>, min_signature_matches: u32, } @@ -147,6 +156,7 @@ impl XskPair { direction: Direction, engine: Option>, rule_engine: Option>, + fusion_engine: Option>, min_signature_matches: u32, ) -> Result { let rx_ifname_c = CString::new(rx_ifname).map_err(|_| SystemError::UnknownError)?; @@ -206,6 +216,7 @@ impl XskPair { frame_pool: Arc::new(Mutex::new(pool_frames)), engine, rule_engine, + fusion_engine, min_signature_matches, }; @@ -340,14 +351,17 @@ impl XskPair { if let Some(r) = &mut *reassembler { match r.process(&packet_data) { Ok(matches) => { - for m in matches { + for m in &matches { log!(RuleLog::SignatureMatch( format!("{:?}", self.direction), - m.src, - m.dst, + m.src.clone(), + m.dst.clone(), m.sid, - m.msg, + m.msg.clone(), )); + if let Some(ref fe) = self.fusion_engine { + fe.record_rule(m); + } } } Err(e) => log!(e), diff --git a/net-guardia/src/core/infrastructure/ml_alert.rs b/net-guardia/src/core/infrastructure/ml_alert.rs deleted file mode 100644 index b9b38ba..0000000 --- a/net-guardia/src/core/infrastructure/ml_alert.rs +++ /dev/null @@ -1,39 +0,0 @@ -use tokio::sync::broadcast; -use tracing::error; - -use crate::model::ml_detection::{AlertMessage, DetectionResult}; - -pub struct MLAlert { - broadcast_tx: broadcast::Sender, -} - -impl MLAlert { - pub fn new() -> Self { - let (broadcast_tx, _) = broadcast::channel(100); - - MLAlert { broadcast_tx } - } - - pub fn subscribe_to_alerts(&self) -> broadcast::Receiver { - self.broadcast_tx.subscribe() - } - - pub fn broadcast_alert(&self, result: &DetectionResult) { - if self.broadcast_tx.receiver_count() > 0 { - let alert = AlertMessage::from_detection_result(result); - if let Err(e) = self.broadcast_tx.send(alert) { - error!("Failed to broadcast ML alert: {}", e); - } - } - } - - pub fn has_subscribers(&self) -> bool { - self.broadcast_tx.receiver_count() > 0 - } -} - -impl Default for MLAlert { - fn default() -> Self { - Self::new() - } -} diff --git a/net-guardia/src/core/infrastructure/mod.rs b/net-guardia/src/core/infrastructure/mod.rs index 3685a20..537ab04 100644 --- a/net-guardia/src/core/infrastructure/mod.rs +++ b/net-guardia/src/core/infrastructure/mod.rs @@ -1,34 +1,35 @@ pub mod app_config; +pub mod detection_alert; pub mod health; pub mod geoip; -pub mod ml_alert; use std::path::PathBuf; - -use chrono::Local; use std::sync::Arc; use std::time::Duration; +use chrono::Local; use crossbeam::queue::SegQueue; use macros::log; use tokio::sync::oneshot; use crate::core::infrastructure::app_config::AppConfig; +use crate::core::infrastructure::detection_alert::DetectionAlert; use crate::core::infrastructure::health::SystemHealth; -use crate::core::infrastructure::ml_alert::MLAlert; +use crate::detection::fusion::{FusionEngine, FusionMode}; use crate::detection::ml::config_loader::InferenceConfig; use crate::detection::ml::engine::Engine; use crate::detection::ml::feature_extractor::FlowFeatures; use crate::detection::ml::model_loader::MLModels; +use crate::detection::ml::traffic_logger::TrafficLogger; use crate::model::error::misc::MiscError; use crate::model::error::Error; use crate::model::error::system::SystemError; use crate::model::log::system::SystemLog; -use crate::detection::ml::traffic_logger::TrafficLogger; pub struct AppServices { pub health: Arc, - pub ml_alert: Arc, + pub detection_alert: Arc, + pub fusion_engine: Arc, pub ml_models: Arc, pub ml_engine: Arc, shutdowns: SegQueue>, @@ -37,9 +38,15 @@ pub struct AppServices { impl AppServices { pub fn new(app_config: Arc, inference_config: Arc) -> Result { let health = SystemHealth::new(app_config.clone())?; - let ml_models = Arc::new(MLModels::load_models(&app_config)?); - let ml_alert = Arc::new(MLAlert::new()); + + let detection_alert = Arc::new(DetectionAlert::new()); + let mode = FusionMode::from_str(&app_config.fusion_mode); + let fusion_engine = Arc::new(FusionEngine::new( + mode, + app_config.fusion_window_secs, + detection_alert.clone(), + )); let traffic_logger = if app_config.traffic_logging_mode { let dir = PathBuf::from(env!("CSV_RECORD_PATH")); @@ -58,7 +65,7 @@ impl AppServices { let ml_engine = Arc::new(Engine::new( ml_models.clone(), inference_config.clone(), - ml_alert.clone(), + fusion_engine.clone(), app_config.max_concurrent_flows, app_config.min_packets_for_inference, app_config.inference_batch_size, @@ -69,10 +76,10 @@ impl AppServices { app_config.ml_cpu, )); - Ok(Self { health: Arc::new(health), - ml_alert, + detection_alert, + fusion_engine, ml_models, ml_engine, shutdowns: SegQueue::new(), @@ -99,4 +106,4 @@ impl AppServices { } } } -} \ No newline at end of file +} diff --git a/net-guardia/src/core/system.rs b/net-guardia/src/core/system.rs index d34545d..73f6a64 100644 --- a/net-guardia/src/core/system.rs +++ b/net-guardia/src/core/system.rs @@ -20,7 +20,7 @@ use crate::model::error::Error; use crate::model::log::ml::MLLog; use crate::model::log::system::SystemLog; use crate::utils::logging::Logging; -use crate::web::api::{control, default, health, misc, ml_alert}; +use crate::web::api::{control, default, detection_alert, health, misc}; pub struct System { pub app_config: Arc, @@ -82,7 +82,7 @@ impl System { log!(SystemLog::InitializeComplete); self.attach_ebpf()?; - ebpf_services.run(app_services.ml_engine.clone()).await?; + ebpf_services.run(app_services.ml_engine.clone(), app_services.fusion_engine.clone()).await?; app_services.run().await?; self.run_http_server().await?; Ok(()) @@ -140,7 +140,7 @@ impl System { let service = self.ebpf_services.service.clone(); let statistics = self.ebpf_services.statistics.clone(); let health = self.app_services.health.clone(); - let ml_alert = self.app_services.ml_alert.clone(); + let detection_alert = self.app_services.detection_alert.clone(); let port = self.app_config.http_server_bind_port; HttpServer::new(move || { let cors = actix_cors::Cors::default() @@ -156,9 +156,9 @@ impl System { .app_data(web::Data::from(service.clone())) .app_data(web::Data::from(statistics.clone())) .app_data(web::Data::from(health.clone())) - .app_data(web::Data::from(ml_alert.clone())) + .app_data(web::Data::from(detection_alert.clone())) .service(control::initialize()) - .service(ml_alert::initialize()) + .service(detection_alert::initialize()) .service(health::initialize()) .service(misc::initialize()) .default_service(route().to(default::default_route)) diff --git a/net-guardia/src/detection/ml/engine.rs b/net-guardia/src/detection/ml/engine.rs index 27ee8c0..b1d2ed8 100644 --- a/net-guardia/src/detection/ml/engine.rs +++ b/net-guardia/src/detection/ml/engine.rs @@ -13,7 +13,7 @@ use super::inference::Inference; use super::model_loader::MLModels; use super::traffic_logger::TrafficLogger; -use crate::core::infrastructure::ml_alert::MLAlert; +use crate::detection::fusion::FusionEngine; use crate::model::error::ml::MLError; use crate::model::log::ml::MLLog; use crate::model::ml_detection::{EngineStats, InferenceStats}; @@ -24,7 +24,7 @@ pub struct Engine { tracker: Arc>, inference_pipeline: Arc, aggregator: Arc>, - ml_alert: Arc, + fusion_engine: Arc, min_packets: usize, batch_size: usize, inference_interval_secs: u64, @@ -37,7 +37,7 @@ impl Engine { pub fn new( models: Arc, config: Arc, - ml_alert: Arc, + fusion_engine: Arc, max_flows: usize, min_packets: usize, batch_size: usize, @@ -59,7 +59,7 @@ impl Engine { tracker, inference_pipeline, aggregator, - ml_alert, + fusion_engine, min_packets: effective_min_packets, batch_size, inference_interval_secs: interval_secs, @@ -202,7 +202,7 @@ impl Engine { result.confidence, result.ae_score, )); - self.ml_alert.broadcast_alert(result); + self.fusion_engine.record_ml(result); } } } diff --git a/net-guardia/src/detection/ml/model_loader.rs b/net-guardia/src/detection/ml/model_loader.rs index 5cd16c5..28441e8 100644 --- a/net-guardia/src/detection/ml/model_loader.rs +++ b/net-guardia/src/detection/ml/model_loader.rs @@ -1,10 +1,12 @@ use std::path::PathBuf; use std::sync::{Arc, Mutex}; +use macros::log; use ort::session::{Session, builder::GraphOptimizationLevel}; use crate::core::infrastructure::app_config::AppConfig; use crate::model::error::ml::MLError; +use crate::model::log::ml::MLLog; use crate::model::ml_detection::RunnableModel; pub struct MLModels { @@ -13,9 +15,17 @@ pub struct MLModels { impl MLModels { pub fn load_models(app_config: &Arc) -> Result { - // Register ort-tract as the ORT execution backend before creating any sessions. - // Returns false only if already initialized, which is harmless. - ort::set_api(ort_tract::api()); + #[cfg(feature = "native-ort-backend")] + { + log!(MLLog::BackendNativeOrt); + ort::init_from(PathBuf::from(env!("ONNXRUNTIME_PATH"))) + .map_err(|_| MLError::InitializeFailed)?.commit(); + } + #[cfg(feature = "tract-backend")] + { + log!(MLLog::BackendTract); + ort::set_api(ort_tract::api()); + } Ok(Self { deep_autoencoder: Mutex::new(Self::load_lstm_ae(&app_config.deep_autoencoder_name)?), diff --git a/net-guardia/src/detection/mod.rs b/net-guardia/src/detection/mod.rs index 7fcb61d..ebe9df7 100644 --- a/net-guardia/src/detection/mod.rs +++ b/net-guardia/src/detection/mod.rs @@ -1,2 +1,3 @@ +pub mod fusion; pub mod ml; pub mod rule; \ No newline at end of file diff --git a/net-guardia/src/model/config.rs b/net-guardia/src/model/config.rs index 0dfa4d6..2479917 100644 --- a/net-guardia/src/model/config.rs +++ b/net-guardia/src/model/config.rs @@ -44,4 +44,20 @@ pub struct Config { /// CPU core pinned to the ML inference spawn_blocking thread. /// If absent, defaults to the last available core. pub ml_cpu: Option, + /// Alert fusion mode: "or" (alert when either source fires) or "and" (require both). + /// Defaults to "or" when absent. + #[serde(default = "default_fusion_mode")] + pub fusion_mode: String, + /// Seconds within which both ML and Rule must fire to be correlated as Fusion. + /// Only used in "or" (corroboration window) and "and" modes. Defaults to 10. + #[serde(default = "default_fusion_window_secs")] + pub fusion_window_secs: u64, +} + +fn default_fusion_mode() -> String { + "or".to_string() +} + +fn default_fusion_window_secs() -> u64 { + 10 } \ No newline at end of file diff --git a/net-guardia/src/model/log/ml.rs b/net-guardia/src/model/log/ml.rs index 9f08bd1..5ecad54 100644 --- a/net-guardia/src/model/log/ml.rs +++ b/net-guardia/src/model/log/ml.rs @@ -6,6 +6,12 @@ loggable! { #[error("Initializing Machine Learning with inference URL: {url}")] Initializing { url: String } => tracing::Level::INFO, + #[error("ML backend: native ORT (libonnxruntime.so)")] + BackendNativeOrt => tracing::Level::INFO, + + #[error("ML backend: ort-tract (pure Rust fallback)")] + BackendTract => tracing::Level::INFO, + #[error("Continuing without Machine Learning detection")] Skiped => tracing::Level::WARN, diff --git a/net-guardia/src/model/ml_detection.rs b/net-guardia/src/model/ml_detection.rs index 51bf809..8bc66fb 100644 --- a/net-guardia/src/model/ml_detection.rs +++ b/net-guardia/src/model/ml_detection.rs @@ -3,6 +3,7 @@ use ort::session::Session; use serde::{Deserialize, Serialize}; use crate::model::direction::Direction; +use crate::model::rule_detection::RuleMatch; use crate::utils::packet_parser::{format_ipv4, format_ipv6}; pub type RunnableModel = Session; @@ -123,7 +124,24 @@ pub struct EngineStats { } #[derive(Debug, Clone, Serialize)] -pub struct AlertMessage { +#[serde(rename_all = "snake_case")] +pub enum AlertSource { + Ml, + Rule, + Fusion, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AlertSeverity { + /// Single-source detection: either ML or Rule fired alone. + High, + /// Both ML and Rule agreed on the same flow within the fusion window. + Critical, +} + +#[derive(Debug, Clone, Serialize)] +pub struct UnifiedAlert { pub timestamp: u64, pub flow_key: String, pub src_ip: String, @@ -131,31 +149,94 @@ pub struct AlertMessage { pub src_port: u16, pub dst_port: u16, pub protocol: u8, + pub source: AlertSource, + pub severity: AlertSeverity, pub is_attack: bool, pub attack_type: Option, pub confidence: f32, pub ae_score: f32, + pub rule_sid: Option, + pub rule_msg: Option, } -impl AlertMessage { - pub fn from_detection_result(result: &DetectionResult) -> Self { - let timestamp = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("system time is after UNIX_EPOCH") - .as_secs(); +fn now_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system time is after UNIX_EPOCH") + .as_secs() +} +fn parse_ip_port(s: &str) -> (String, u16) { + match s.rfind(':') { + Some(pos) => { + let port = s[pos + 1..].parse::().unwrap_or(0); + (s[..pos].to_string(), port) + } + None => (s.to_string(), 0), + } +} + +impl UnifiedAlert { + pub fn from_ml(result: &DetectionResult) -> Self { Self { - timestamp, + timestamp: now_secs(), flow_key: result.flow_key.clone(), src_ip: result.flow_key_raw.src_ip.clone(), dst_ip: result.flow_key_raw.dst_ip.clone(), src_port: result.flow_key_raw.src_port, dst_port: result.flow_key_raw.dst_port, protocol: result.flow_key_raw.protocol, + source: AlertSource::Ml, + severity: AlertSeverity::High, is_attack: result.is_attack, attack_type: result.attack_type.clone(), confidence: result.confidence, ae_score: result.ae_score, + rule_sid: None, + rule_msg: None, + } + } + + pub fn from_rule(m: &RuleMatch) -> Self { + let (src_ip, src_port) = parse_ip_port(&m.src); + let (dst_ip, dst_port) = parse_ip_port(&m.dst); + let flow_key = format!("{}->{}", m.src, m.dst); + Self { + timestamp: now_secs(), + flow_key, + src_ip, + dst_ip, + src_port, + dst_port, + protocol: 6, + source: AlertSource::Rule, + severity: AlertSeverity::High, + is_attack: true, + attack_type: None, + confidence: 1.0, + ae_score: 0.0, + rule_sid: Some(m.sid), + rule_msg: Some(m.msg.clone()), + } + } + + pub fn from_fusion(result: &DetectionResult, m: &RuleMatch) -> Self { + Self { + timestamp: now_secs(), + flow_key: result.flow_key.clone(), + src_ip: result.flow_key_raw.src_ip.clone(), + dst_ip: result.flow_key_raw.dst_ip.clone(), + src_port: result.flow_key_raw.src_port, + dst_port: result.flow_key_raw.dst_port, + protocol: result.flow_key_raw.protocol, + source: AlertSource::Fusion, + severity: AlertSeverity::Critical, + is_attack: result.is_attack, + attack_type: result.attack_type.clone(), + confidence: result.confidence, + ae_score: result.ae_score, + rule_sid: Some(m.sid), + rule_msg: Some(m.msg.clone()), } } } \ No newline at end of file diff --git a/net-guardia/src/web/api/mod.rs b/net-guardia/src/web/api/mod.rs index df7dce0..10cdba6 100644 --- a/net-guardia/src/web/api/mod.rs +++ b/net-guardia/src/web/api/mod.rs @@ -1,5 +1,5 @@ pub mod control; pub mod default; -pub mod misc; -pub mod ml_alert; +pub mod detection_alert; pub mod health; +pub mod misc; diff --git a/net-guardia/src/web/websocket/alert_websocket.rs b/net-guardia/src/web/websocket/alert_websocket.rs index cdff3d4..d029fcb 100644 --- a/net-guardia/src/web/websocket/alert_websocket.rs +++ b/net-guardia/src/web/websocket/alert_websocket.rs @@ -4,8 +4,8 @@ use futures_util::StreamExt; use macros::log; use tokio::sync::broadcast; -use crate::core::infrastructure::ml_alert::MLAlert; -use crate::model::ml_detection::AlertMessage; +use crate::core::infrastructure::detection_alert::DetectionAlert; +use crate::model::ml_detection::UnifiedAlert; use crate::model::error::http::HttpError; use crate::model::error::misc::MiscError; use crate::model::log::http::HttpLog; @@ -13,11 +13,11 @@ use crate::model::log::http::HttpLog; pub async fn websocket_alert( req: HttpRequest, body: web::Payload, - ai: web::Data, + da: web::Data, ) -> Result { let (response, session, msg_stream) = handle(&req, body)?; - let broadcast_rx = ai.subscribe_to_alerts(); + let broadcast_rx = da.subscribe(); actix_web::rt::spawn(async move { handle_alert_connection(session, msg_stream, broadcast_rx).await; @@ -29,7 +29,7 @@ pub async fn websocket_alert( async fn handle_alert_connection( mut session: Session, mut msg_stream: MessageStream, - mut broadcast_rx: broadcast::Receiver, + mut broadcast_rx: broadcast::Receiver, ) { loop { tokio::select! { @@ -80,7 +80,7 @@ async fn handle_client_message( } } -async fn send_alert(session: &mut Session, alert: &AlertMessage) -> bool { +async fn send_alert(session: &mut Session, alert: &UnifiedAlert) -> bool { match serde_json::to_string(alert) { Ok(json) => session.text(json).await.is_ok(), Err(err) => { @@ -88,4 +88,4 @@ async fn send_alert(session: &mut Session, alert: &AlertMessage) -> bool { false } } -} \ No newline at end of file +} diff --git a/onnxruntime/info.txt b/onnxruntime/info.txt new file mode 100644 index 0000000..757ba0f --- /dev/null +++ b/onnxruntime/info.txt @@ -0,0 +1,2 @@ +// this's onnxruntime library folder, so you can download the library from https://github.com/ParrotXray/onnxruntime-builder/releases +// you can select that matches the system architecture \ No newline at end of file