mirror of
https://github.com/DaLaw2/NetGuardia.git
synced 2026-08-24 14:10:28 +09:00
feat: batched ONNX inference, per-model observability, circuit breaker
T11: Batched ONNX inference
- model_loader compiles models with [batch_size, features] shape
- inference.rs builds (B, N) tensors, runs AE and classifier once per chunk
- Zero-padding for partial batches, chunking for overflow
- Cache C2/Normal class indices at construction
- ~N/batch_size × 2 ONNX dispatches instead of N × 2
T13: Per-model observability
- Add anomaly_score and c2_score to DetectionResult, AlertMessage,
DetectionEvent, ThreatDetectedEvent
- Scores flow through: inference → alert → ML bridge → orchestrator → SOAR
- DetectionEmitted log includes ae/anomaly/c2 scores
- SOAR ActionLog includes per-model scores for FP debugging
T14: Circuit breaker for failing models
- Track consecutive failures with AtomicU32 (lock-free)
- 5 failures within 60s window trips the breaker
- 120s cooldown before auto-reset
- Failures recorded on: batch panic, AE error, classifier error
- Success resets counter immediately
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
4bfb01c1a2
commit
8d020c27bd
@ -91,6 +91,9 @@ impl BotnetDetector {
|
||||
protocol: alert.protocol,
|
||||
packet_count: 0,
|
||||
flow_duration_us: 0,
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let _ = detection_tx.try_send(event);
|
||||
@ -142,6 +145,8 @@ mod tests {
|
||||
attack_type: Some("DDoS".to_string()),
|
||||
confidence: 0.9,
|
||||
ae_score: 0.5,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
packet_count: 100,
|
||||
flow_duration_us: 1_000_000,
|
||||
}
|
||||
|
||||
@ -89,6 +89,9 @@ impl LateralMovementDetector {
|
||||
protocol: alert.protocol,
|
||||
packet_count: 0,
|
||||
flow_duration_us: 0,
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let _ = detection_tx.try_send(event);
|
||||
@ -201,6 +204,8 @@ mod tests {
|
||||
attack_type: Some("Exploitation".to_string()),
|
||||
confidence: 0.8,
|
||||
ae_score: 0.4,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
packet_count: 50,
|
||||
flow_duration_us: 500_000,
|
||||
}
|
||||
|
||||
@ -87,6 +87,9 @@ impl ScanDetector {
|
||||
protocol: alert.protocol,
|
||||
packet_count: 0,
|
||||
flow_duration_us: 0,
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let _ = detection_tx.try_send(event);
|
||||
@ -137,6 +140,8 @@ mod tests {
|
||||
attack_type: Some("Reconnaissance".to_string()),
|
||||
confidence: 0.7,
|
||||
ae_score: 0.3,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
packet_count: 5,
|
||||
flow_duration_us: 100_000,
|
||||
}
|
||||
|
||||
@ -141,6 +141,9 @@ impl BeaconingDetector {
|
||||
protocol: 6,
|
||||
packet_count: count as u64,
|
||||
flow_duration_us: 0,
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let _ = self.detection_tx.try_send(event);
|
||||
|
||||
@ -117,6 +117,9 @@ impl DetectionOrchestrator {
|
||||
source_ip: event.source_ip.clone(),
|
||||
attack_type: event.attack_type.clone(),
|
||||
confidence: event.confidence,
|
||||
ae_score: event.ae_score,
|
||||
anomaly_score: event.anomaly_score,
|
||||
c2_score: event.c2_score,
|
||||
sources_count: sources.len(),
|
||||
});
|
||||
|
||||
@ -187,6 +190,9 @@ impl DetectionOrchestrator {
|
||||
geoip_country,
|
||||
is_repeat_offender: is_repeat,
|
||||
sources: vec![event.source.clone()],
|
||||
ae_score: event.ae_score,
|
||||
anomaly_score: event.anomaly_score,
|
||||
c2_score: event.c2_score,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use macros::log;
|
||||
use tract_onnx::prelude::*;
|
||||
@ -10,97 +12,242 @@ use crate::model::detection::ml_detection::DetectionResult;
|
||||
use crate::model::log::ml::MLLog;
|
||||
use crate::model::system::config::MLInferenceConfig;
|
||||
|
||||
/// (anomaly_scores, per_class_probs, c2_scores)
|
||||
type ClassifierBatchOutput = (Vec<f32>, Vec<Vec<f32>>, Vec<f32>);
|
||||
|
||||
/// Consecutive failures to trip the circuit breaker.
|
||||
const CIRCUIT_BREAKER_THRESHOLD: u32 = 5;
|
||||
/// Window in seconds: failures older than this are forgotten.
|
||||
const CIRCUIT_BREAKER_WINDOW_SECS: u64 = 60;
|
||||
/// Cooldown in seconds before re-enabling inference after circuit break.
|
||||
const CIRCUIT_BREAKER_COOLDOWN_SECS: u64 = 120;
|
||||
|
||||
pub struct Inference {
|
||||
pub models: Arc<MLModels>,
|
||||
pub config: Arc<MLInferenceConfig>,
|
||||
c2_class_idx: Option<usize>,
|
||||
normal_class_idx: Option<usize>,
|
||||
/// Consecutive failure count within the current window.
|
||||
failure_count: AtomicU32,
|
||||
/// Timestamp (epoch secs) of first failure in current window. 0 = no failures.
|
||||
failure_window_start: AtomicU64,
|
||||
/// Timestamp (epoch secs) when circuit breaker tripped. 0 = not tripped.
|
||||
circuit_open_since: AtomicU64,
|
||||
}
|
||||
|
||||
impl Inference {
|
||||
pub fn new(models: Arc<MLModels>, config: Arc<MLInferenceConfig>) -> Self {
|
||||
Self { models, config }
|
||||
let c2_class_idx = Self::find_label_index(&config, "C2 Communication");
|
||||
let normal_class_idx = Self::find_label_index(&config, "Normal");
|
||||
Self {
|
||||
models,
|
||||
config,
|
||||
c2_class_idx,
|
||||
normal_class_idx,
|
||||
failure_count: AtomicU32::new(0),
|
||||
failure_window_start: AtomicU64::new(0),
|
||||
circuit_open_since: AtomicU64::new(0),
|
||||
}
|
||||
}
|
||||
|
||||
fn find_label_index(config: &MLInferenceConfig, label: &str) -> Option<usize> {
|
||||
config
|
||||
.attack_labels
|
||||
.iter()
|
||||
.find(|(_, v)| v.as_str() == label)
|
||||
.and_then(|(k, _)| k.parse::<usize>().ok())
|
||||
}
|
||||
|
||||
/// Batched inference with circuit breaker protection.
|
||||
pub fn infer_batch(&self, flows: &[FlowData]) -> Vec<DetectionResult> {
|
||||
flows.iter().filter_map(|flow| self.infer_single(flow)).collect()
|
||||
}
|
||||
if self.is_circuit_open() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
pub fn infer_single(&self, flow: &FlowData) -> Option<DetectionResult> {
|
||||
// catch_unwind protects against tract-onnx internal panics on edge-case inputs.
|
||||
// Without this, panic=abort config would kill the entire process.
|
||||
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.infer_single_inner(flow))) {
|
||||
Ok(result) => result,
|
||||
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.infer_batch_inner(flows))) {
|
||||
Ok(results) => {
|
||||
// Success: reset failure counter
|
||||
self.failure_count.store(0, Ordering::Relaxed);
|
||||
self.failure_window_start.store(0, Ordering::Relaxed);
|
||||
results
|
||||
}
|
||||
Err(_) => {
|
||||
log!(MLLog::InferenceFailed(
|
||||
"ONNX".to_string(),
|
||||
"inference panicked (caught)".to_string(),
|
||||
"batch inference panicked (caught)".to_string(),
|
||||
));
|
||||
None
|
||||
self.record_failure();
|
||||
Vec::new()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn infer_single_inner(&self, flow: &FlowData) -> Option<DetectionResult> {
|
||||
let ae_features = self.preprocess_ae_features(flow);
|
||||
fn infer_batch_inner(&self, flows: &[FlowData]) -> Vec<DetectionResult> {
|
||||
let n = flows.len();
|
||||
let batch_size = self.models.batch_size;
|
||||
let n_ae = self.config.num_ae_features();
|
||||
let n_cls = self.config.num_classifier_features();
|
||||
|
||||
let ae_input = Self::vec_to_array2(&ae_features);
|
||||
let ae_score = match self.run_autoencoder(&ae_input) {
|
||||
Ok(score) => score,
|
||||
Err(e) => {
|
||||
log!(MLLog::InferenceFailed("DeepAutoEncoder".to_string(), e.to_string()));
|
||||
return None;
|
||||
// Phase 1: preprocess all AE features
|
||||
let all_ae_features: Vec<Vec<f32>> = flows.iter().map(|f| self.preprocess_ae_features(f)).collect();
|
||||
|
||||
// Phase 2: run AE in chunks → compute per-flow MSE
|
||||
let mut ae_scores = Vec::with_capacity(n);
|
||||
for chunk_start in (0..n).step_by(batch_size) {
|
||||
let chunk_end = (chunk_start + batch_size).min(n);
|
||||
let actual = chunk_end - chunk_start;
|
||||
|
||||
// Build (batch_size, n_ae) tensor, zero-padded
|
||||
let ae_input = tract_ndarray::Array2::<f32>::from_shape_fn((batch_size, n_ae), |(i, j)| {
|
||||
if i < actual {
|
||||
all_ae_features[chunk_start + i][j]
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
});
|
||||
|
||||
match self.run_ae_batch(&ae_input, actual, n_ae) {
|
||||
Ok(scores) => ae_scores.extend_from_slice(&scores),
|
||||
Err(e) => {
|
||||
log!(MLLog::InferenceFailed("DeepAutoEncoder".to_string(), e.to_string()));
|
||||
self.record_failure();
|
||||
return Vec::new();
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
let cls_input = self.build_classifier_input(&ae_features, ae_score);
|
||||
// Phase 3: build classifier input (ae_features ++ ae_score) and run in chunks
|
||||
let mut all_anomaly = Vec::with_capacity(n);
|
||||
let mut all_class_probs = Vec::with_capacity(n);
|
||||
let mut all_c2_scores = Vec::with_capacity(n);
|
||||
|
||||
let flow_key = format!(
|
||||
"{}:{} -> {}:{} (proto {}) [{}]",
|
||||
flow.flow_key.src_ip_string(),
|
||||
flow.flow_key.src_port,
|
||||
flow.flow_key.dst_ip_string(),
|
||||
flow.flow_key.dst_port,
|
||||
flow.flow_key.protocol,
|
||||
flow.direction
|
||||
);
|
||||
for chunk_start in (0..n).step_by(batch_size) {
|
||||
let chunk_end = (chunk_start + batch_size).min(n);
|
||||
let actual = chunk_end - chunk_start;
|
||||
|
||||
let result = match self.models.classifier.run(tvec![cls_input.into_tensor().into()]) {
|
||||
Ok(r) => r,
|
||||
Err(e) => {
|
||||
log!(MLLog::InferenceFailed("MultiTaskModel".to_string(), e.to_string()));
|
||||
return None;
|
||||
// Build (batch_size, n_cls) tensor
|
||||
let cls_input = tract_ndarray::Array2::<f32>::from_shape_fn((batch_size, n_cls), |(i, j)| {
|
||||
if i < actual {
|
||||
if j < n_ae {
|
||||
all_ae_features[chunk_start + i][j]
|
||||
} else {
|
||||
ae_scores[chunk_start + i]
|
||||
}
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
});
|
||||
|
||||
match self.run_classifier_batch(&cls_input, actual) {
|
||||
Ok((anomaly, class_probs, c2)) => {
|
||||
all_anomaly.extend_from_slice(&anomaly);
|
||||
all_class_probs.extend(class_probs);
|
||||
all_c2_scores.extend_from_slice(&c2);
|
||||
}
|
||||
Err(e) => {
|
||||
log!(MLLog::InferenceFailed("MultiTaskModel".to_string(), e.to_string()));
|
||||
self.record_failure();
|
||||
return Vec::new();
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Output 0: anomaly (shape [1,1], sigmoid)
|
||||
let anomaly_score = match result[0].to_array_view::<f32>() {
|
||||
Ok(v) => *v.iter().next().unwrap_or(&0.0),
|
||||
Err(e) => {
|
||||
log!(MLLog::InferenceFailed("anomaly_head".to_string(), e.to_string()));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
// Phase 4: build DetectionResults
|
||||
let mut results = Vec::with_capacity(n);
|
||||
for i in 0..n {
|
||||
let flow = &flows[i];
|
||||
let flow_key = format!(
|
||||
"{}:{} -> {}:{} (proto {}) [{}]",
|
||||
flow.flow_key.src_ip_string(),
|
||||
flow.flow_key.src_port,
|
||||
flow.flow_key.dst_ip_string(),
|
||||
flow.flow_key.dst_port,
|
||||
flow.flow_key.protocol,
|
||||
flow.direction
|
||||
);
|
||||
|
||||
// Output 1: class_probs (shape [1, n_classes], softmax)
|
||||
let class_probs = match result[1].to_array_view::<f32>() {
|
||||
Ok(v) => v.iter().copied().collect::<Vec<f32>>(),
|
||||
Err(e) => {
|
||||
log!(MLLog::InferenceFailed("class_head".to_string(), e.to_string()));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let result = self.build_detection_result(
|
||||
flow,
|
||||
flow_key,
|
||||
ae_scores[i],
|
||||
all_anomaly[i],
|
||||
&all_class_probs[i],
|
||||
all_c2_scores[i],
|
||||
);
|
||||
results.push(result);
|
||||
}
|
||||
|
||||
// Output 2: c2_score (shape [1,1], sigmoid)
|
||||
let c2_score = match result[2].to_array_view::<f32>() {
|
||||
Ok(v) => *v.iter().next().unwrap_or(&0.0),
|
||||
Err(e) => {
|
||||
log!(MLLog::InferenceFailed("c2_head".to_string(), e.to_string()));
|
||||
return None;
|
||||
}
|
||||
};
|
||||
results
|
||||
}
|
||||
|
||||
/// Run AE on a padded batch, return MSE scores for the first `actual` rows.
|
||||
fn run_ae_batch(
|
||||
&self,
|
||||
input: &tract_ndarray::Array2<f32>,
|
||||
actual: usize,
|
||||
n_features: usize,
|
||||
) -> TractResult<Vec<f32>> {
|
||||
let result = self.models.deep_autoencoder.run(tvec![input.clone().into_tensor().into()])?;
|
||||
|
||||
let output = result[0]
|
||||
.to_array_view::<f32>()?
|
||||
.into_dimensionality::<tract_ndarray::Ix2>()?;
|
||||
|
||||
let diff = input - &output;
|
||||
let sq = &diff * &diff;
|
||||
|
||||
let mut scores = Vec::with_capacity(actual);
|
||||
let n_f = n_features as f32;
|
||||
for i in 0..actual {
|
||||
let mse: f32 = sq.row(i).sum() / n_f;
|
||||
scores.push(mse);
|
||||
}
|
||||
Ok(scores)
|
||||
}
|
||||
|
||||
/// Run multi-task classifier on a padded batch, return (anomaly, class_probs, c2) for first `actual` rows.
|
||||
/// Returns (anomaly_scores, per_class_probs, c2_scores) for the first `actual` rows.
|
||||
fn run_classifier_batch(
|
||||
&self,
|
||||
input: &tract_ndarray::Array2<f32>,
|
||||
actual: usize,
|
||||
) -> TractResult<ClassifierBatchOutput> {
|
||||
let result = self.models.classifier.run(tvec![input.clone().into_tensor().into()])?;
|
||||
|
||||
// Output 0: anomaly (batch_size, 1)
|
||||
let anomaly_view = result[0].to_array_view::<f32>()?;
|
||||
let anomaly: Vec<f32> = (0..actual)
|
||||
.map(|i| anomaly_view.as_slice().map(|s| s[i]).unwrap_or(0.0))
|
||||
.collect();
|
||||
|
||||
// Output 1: class_probs (batch_size, n_classes)
|
||||
let class_view = result[1]
|
||||
.to_array_view::<f32>()?
|
||||
.into_dimensionality::<tract_ndarray::Ix2>()?;
|
||||
let class_probs: Vec<Vec<f32>> = (0..actual)
|
||||
.map(|i| class_view.row(i).iter().copied().collect())
|
||||
.collect();
|
||||
|
||||
// Output 2: c2_score (batch_size, 1)
|
||||
let c2_view = result[2].to_array_view::<f32>()?;
|
||||
let c2: Vec<f32> = (0..actual)
|
||||
.map(|i| c2_view.as_slice().map(|s| s[i]).unwrap_or(0.0))
|
||||
.collect();
|
||||
|
||||
Ok((anomaly, class_probs, c2))
|
||||
}
|
||||
|
||||
fn build_detection_result(
|
||||
&self,
|
||||
flow: &FlowData,
|
||||
flow_key: String,
|
||||
ae_score: f32,
|
||||
anomaly_score: f32,
|
||||
class_probs: &[f32],
|
||||
c2_score: f32,
|
||||
) -> DetectionResult {
|
||||
let is_attack = anomaly_score > self.config.anomaly_threshold;
|
||||
|
||||
// Determine attack type from class_head argmax
|
||||
let (predicted_class, class_confidence) = class_probs
|
||||
.iter()
|
||||
.enumerate()
|
||||
@ -117,19 +264,21 @@ impl Inference {
|
||||
|
||||
let mut confidence = class_confidence;
|
||||
|
||||
// C2 head override: if C2 head fires and its confidence exceeds the class head's
|
||||
// C2 probability, prefer the dedicated C2 head judgment.
|
||||
// C2 head override
|
||||
if c2_score > self.config.c2_threshold {
|
||||
let c2_class_prob = self.find_class_prob_for("C2 Communication", &class_probs);
|
||||
let c2_class_prob = self
|
||||
.c2_class_idx
|
||||
.and_then(|idx| class_probs.get(idx).copied())
|
||||
.unwrap_or(0.0);
|
||||
if c2_score > c2_class_prob {
|
||||
attack_type = "C2 Communication".to_string();
|
||||
confidence = c2_score;
|
||||
}
|
||||
}
|
||||
|
||||
// "Normal" class prediction means benign
|
||||
if attack_type == "Normal" {
|
||||
return Some(DetectionResult {
|
||||
// "Normal" class = benign
|
||||
if Some(predicted_class) == self.normal_class_idx && c2_score <= self.config.c2_threshold {
|
||||
return DetectionResult {
|
||||
flow_key,
|
||||
flow_key_raw: flow.flow_key.clone(),
|
||||
direction: flow.direction,
|
||||
@ -137,13 +286,15 @@ impl Inference {
|
||||
attack_type: None,
|
||||
confidence: class_confidence,
|
||||
ae_score,
|
||||
anomaly_score,
|
||||
c2_score,
|
||||
threshold: self.config.anomaly_threshold,
|
||||
packet_count: flow.packet_count() as u64,
|
||||
flow_duration_us: flow.duration_us(),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
Some(DetectionResult {
|
||||
DetectionResult {
|
||||
flow_key,
|
||||
flow_key_raw: flow.flow_key.clone(),
|
||||
direction: flow.direction,
|
||||
@ -151,20 +302,12 @@ impl Inference {
|
||||
attack_type: if is_attack { Some(attack_type) } else { None },
|
||||
confidence,
|
||||
ae_score,
|
||||
anomaly_score,
|
||||
c2_score,
|
||||
threshold: self.config.anomaly_threshold,
|
||||
packet_count: flow.packet_count() as u64,
|
||||
flow_duration_us: flow.duration_us(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Find the softmax probability for a given label name in class_probs.
|
||||
fn find_class_prob_for(&self, label: &str, class_probs: &[f32]) -> f32 {
|
||||
for (key, name) in &self.config.attack_labels {
|
||||
if name == label && let Ok(idx) = key.parse::<usize>() {
|
||||
return class_probs.get(idx).copied().unwrap_or(0.0);
|
||||
}
|
||||
}
|
||||
0.0
|
||||
}
|
||||
|
||||
fn preprocess_ae_features(&self, flow: &FlowData) -> Vec<f32> {
|
||||
@ -175,33 +318,47 @@ impl Inference {
|
||||
features.features.iter().map(|&x| x as f32).collect()
|
||||
}
|
||||
|
||||
fn vec_to_array2(v: &[f32]) -> tract_ndarray::Array2<f32> {
|
||||
tract_ndarray::Array2::from_shape_fn((1, v.len()), |(_, j)| v[j])
|
||||
// -- Circuit breaker ---------------------------------------------------------
|
||||
|
||||
fn now_secs() -> u64 {
|
||||
SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Classifier input = preprocessed ae_features ++ [ae_anomaly_score]
|
||||
fn build_classifier_input(&self, ae_features: &[f32], ae_score: f32) -> tract_ndarray::Array2<f32> {
|
||||
let n = ae_features.len() + 1;
|
||||
tract_ndarray::Array2::from_shape_fn((1, n), |(_, j)| {
|
||||
if j < ae_features.len() {
|
||||
ae_features[j]
|
||||
} else {
|
||||
ae_score
|
||||
}
|
||||
})
|
||||
fn is_circuit_open(&self) -> bool {
|
||||
let open_since = self.circuit_open_since.load(Ordering::Relaxed);
|
||||
if open_since == 0 {
|
||||
return false;
|
||||
}
|
||||
let elapsed = Self::now_secs().saturating_sub(open_since);
|
||||
if elapsed >= CIRCUIT_BREAKER_COOLDOWN_SECS {
|
||||
// Cooldown elapsed — reset and allow inference
|
||||
self.circuit_open_since.store(0, Ordering::Relaxed);
|
||||
self.failure_count.store(0, Ordering::Relaxed);
|
||||
self.failure_window_start.store(0, Ordering::Relaxed);
|
||||
log!(MLLog::CircuitBreakerReset(CIRCUIT_BREAKER_COOLDOWN_SECS));
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn run_autoencoder(&self, input: &tract_ndarray::Array2<f32>) -> TractResult<f32> {
|
||||
let input_tensor = input.clone().into_tensor();
|
||||
let result = self.models.deep_autoencoder.run(tvec![input_tensor.into()])?;
|
||||
fn record_failure(&self) {
|
||||
let now = Self::now_secs();
|
||||
let window_start = self.failure_window_start.load(Ordering::Relaxed);
|
||||
|
||||
let output = result[0]
|
||||
.to_array_view::<f32>()?
|
||||
.into_dimensionality::<tract_ndarray::Ix2>()?;
|
||||
// If window has expired, start a new window
|
||||
if window_start == 0 || now.saturating_sub(window_start) > CIRCUIT_BREAKER_WINDOW_SECS {
|
||||
self.failure_window_start.store(now, Ordering::Relaxed);
|
||||
self.failure_count.store(1, Ordering::Relaxed);
|
||||
return;
|
||||
}
|
||||
|
||||
let diff = input - &output;
|
||||
let mse = (&diff * &diff).sum() / self.config.ae_feature_names.len() as f32;
|
||||
|
||||
Ok(mse)
|
||||
let count = self.failure_count.fetch_add(1, Ordering::Relaxed) + 1;
|
||||
if count >= CIRCUIT_BREAKER_THRESHOLD {
|
||||
self.circuit_open_since.store(now, Ordering::Relaxed);
|
||||
log!(MLLog::CircuitBreakerOpen(count, CIRCUIT_BREAKER_WINDOW_SECS));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -2,35 +2,43 @@ use std::path::PathBuf;
|
||||
use tract_onnx::prelude::*;
|
||||
|
||||
use crate::infrastructure::app_config::AppConfig;
|
||||
use crate::model::error::ml::MLError;
|
||||
use crate::model::detection::ml_detection::RunnableModel;
|
||||
|
||||
use crate::model::error::ml::MLError;
|
||||
use crate::model::system::config::MLInferenceConfig;
|
||||
|
||||
pub struct MLModels {
|
||||
pub deep_autoencoder: RunnableModel,
|
||||
pub classifier: RunnableModel,
|
||||
pub batch_size: usize,
|
||||
}
|
||||
|
||||
impl MLModels {
|
||||
pub fn load_models(app_config: &Arc<AppConfig>, inference_config: &Arc<MLInferenceConfig>) -> Result<Self, MLError> {
|
||||
pub fn load_models(
|
||||
app_config: &Arc<AppConfig>,
|
||||
inference_config: &Arc<MLInferenceConfig>,
|
||||
batch_size: usize,
|
||||
) -> Result<Self, MLError> {
|
||||
Ok(Self {
|
||||
deep_autoencoder: Self::loader(
|
||||
&app_config.inference.deep_autoencoder_name,
|
||||
inference_config.num_ae_features(),
|
||||
batch_size,
|
||||
)?,
|
||||
classifier: Self::loader(
|
||||
&app_config.inference.classifier_name,
|
||||
inference_config.num_classifier_features(),
|
||||
batch_size,
|
||||
)?,
|
||||
batch_size,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn loader(model: &str, features: usize) -> Result<RunnableModel, MLError> {
|
||||
fn loader(model: &str, features: usize, batch_size: usize) -> Result<RunnableModel, MLError> {
|
||||
let model_path = PathBuf::from("models").join(model);
|
||||
|
||||
let load = || -> Result<RunnableModel, Box<dyn std::error::Error>> {
|
||||
let mut model = onnx().model_for_path(&model_path)?;
|
||||
model.set_input_fact(0, f32::fact([1, features]).into())?;
|
||||
model.set_input_fact(0, f32::fact([batch_size, features]).into())?;
|
||||
Ok(model.into_optimized()?.into_runnable()?)
|
||||
};
|
||||
|
||||
@ -46,6 +54,6 @@ impl MLModels {
|
||||
|
||||
let inputs = model.model().inputs.len();
|
||||
let outputs = model.model().outputs.len();
|
||||
format!("{}: inputs: {}, outputs: {}", name, inputs, outputs)
|
||||
format!("{name}: inputs: {inputs}, outputs: {outputs}, batch_size: {}", self.batch_size)
|
||||
}
|
||||
}
|
||||
|
||||
@ -905,6 +905,9 @@ impl SoarEngine {
|
||||
event.source_ip.clone(),
|
||||
event.attack_type.clone(),
|
||||
format!("{:.2}", event.confidence),
|
||||
format!("{:.3}", event.ae_score),
|
||||
format!("{:.3}", event.anomaly_score),
|
||||
format!("{:.3}", event.c2_score),
|
||||
));
|
||||
|
||||
Ok(format!("Logged at level '{}'", level))
|
||||
@ -1215,6 +1218,9 @@ mod tests {
|
||||
geoip_country: None,
|
||||
is_repeat_offender: false,
|
||||
sources: vec![crate::model::event::DetectionSource::ML],
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let action = PlaybookAction {
|
||||
@ -1247,6 +1253,9 @@ mod tests {
|
||||
geoip_country: None,
|
||||
is_repeat_offender: false,
|
||||
sources: vec![crate::model::event::DetectionSource::ML],
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let action = PlaybookAction {
|
||||
@ -1326,6 +1335,9 @@ mod tests {
|
||||
geoip_country: None,
|
||||
is_repeat_offender: false,
|
||||
sources: vec![crate::model::event::DetectionSource::ML],
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let action = PlaybookAction {
|
||||
@ -1358,6 +1370,9 @@ mod tests {
|
||||
geoip_country: None,
|
||||
is_repeat_offender: false,
|
||||
sources: vec![crate::model::event::DetectionSource::ML],
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
// Find a matching playbook — default "threat_detected" playbook should exist
|
||||
@ -1400,6 +1415,9 @@ mod tests {
|
||||
geoip_country: None,
|
||||
is_repeat_offender: false,
|
||||
sources: vec![crate::model::event::DetectionSource::ML],
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
};
|
||||
|
||||
let playbooks = engine.find_matching_playbooks(&event);
|
||||
@ -1473,6 +1491,9 @@ mod tests {
|
||||
geoip_country: country.map(|s| s.to_string()),
|
||||
is_repeat_offender: repeat,
|
||||
sources: vec![crate::model::event::DetectionSource::ML],
|
||||
ae_score: 0.0,
|
||||
anomaly_score: 0.0,
|
||||
c2_score: 0.0,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -403,6 +403,9 @@ impl System {
|
||||
protocol: alert.protocol,
|
||||
packet_count: alert.packet_count,
|
||||
flow_duration_us: alert.flow_duration_us,
|
||||
ae_score: alert.ae_score,
|
||||
anomaly_score: alert.anomaly_score,
|
||||
c2_score: alert.c2_score,
|
||||
};
|
||||
if tx.send(event).await.is_err() {
|
||||
break; // Orchestrator dropped
|
||||
|
||||
@ -41,7 +41,8 @@ impl AppServices {
|
||||
) -> Result<Self, Error> {
|
||||
let health = SystemHealth::new(app_config.clone())?;
|
||||
|
||||
let ml_models = Arc::new(MLModels::load_models(&app_config, &inference_config)?);
|
||||
let batch_size = app_config.inference.inference_batch_size;
|
||||
let ml_models = Arc::new(MLModels::load_models(&app_config, &inference_config, batch_size)?);
|
||||
let ml_alert = Arc::new(MLAlert::new());
|
||||
|
||||
let traffic_logger = if app_config.inference.traffic_logging_mode {
|
||||
|
||||
@ -104,6 +104,8 @@ pub struct DetectionResult {
|
||||
pub attack_type: Option<String>,
|
||||
pub confidence: f32,
|
||||
pub ae_score: f32,
|
||||
pub anomaly_score: f32,
|
||||
pub c2_score: f32,
|
||||
pub threshold: f32,
|
||||
pub packet_count: u64,
|
||||
pub flow_duration_us: u64,
|
||||
@ -151,6 +153,8 @@ pub struct AlertMessage {
|
||||
pub attack_type: Option<String>,
|
||||
pub confidence: f32,
|
||||
pub ae_score: f32,
|
||||
pub anomaly_score: f32,
|
||||
pub c2_score: f32,
|
||||
pub packet_count: u64,
|
||||
pub flow_duration_us: u64,
|
||||
}
|
||||
@ -174,6 +178,8 @@ impl AlertMessage {
|
||||
attack_type: result.attack_type.clone(),
|
||||
confidence: result.confidence,
|
||||
ae_score: result.ae_score,
|
||||
anomaly_score: result.anomaly_score,
|
||||
c2_score: result.c2_score,
|
||||
packet_count: result.packet_count,
|
||||
flow_duration_us: result.flow_duration_us,
|
||||
}
|
||||
|
||||
@ -38,6 +38,10 @@ pub struct DetectionEvent {
|
||||
pub protocol: u8,
|
||||
pub packet_count: u64,
|
||||
pub flow_duration_us: u64,
|
||||
/// Per-model scores for observability (ML source only)
|
||||
pub ae_score: f32,
|
||||
pub anomaly_score: f32,
|
||||
pub c2_score: f32,
|
||||
}
|
||||
|
||||
// -- Threat Events ------------------------------------------------------------
|
||||
@ -64,6 +68,10 @@ pub struct ThreatDetectedEvent {
|
||||
pub is_repeat_offender: bool,
|
||||
/// Which detection sources contributed to this threat (for attribution)
|
||||
pub sources: Vec<DetectionSource>,
|
||||
/// Per-model scores for debugging false positives
|
||||
pub ae_score: f32,
|
||||
pub anomaly_score: f32,
|
||||
pub c2_score: f32,
|
||||
}
|
||||
|
||||
impl Event for ThreatDetectedEvent {}
|
||||
|
||||
@ -9,8 +9,8 @@ loggable! {
|
||||
#[error("Detection deduplicated: {source_ip} {attack_type} (within window)")]
|
||||
DetectionDeduplicated { source_ip: String, attack_type: String } => tracing::Level::DEBUG,
|
||||
|
||||
#[error("Detection emitted: {source_ip} {attack_type} confidence={confidence:.2} sources={sources_count}")]
|
||||
DetectionEmitted { source_ip: String, attack_type: String, confidence: f32, sources_count: usize } => tracing::Level::DEBUG,
|
||||
#[error("Detection emitted: {source_ip} {attack_type} confidence={confidence:.2} ae={ae_score:.3} anomaly={anomaly_score:.3} c2={c2_score:.3} sources={sources_count}")]
|
||||
DetectionEmitted { source_ip: String, attack_type: String, confidence: f32, ae_score: f32, anomaly_score: f32, c2_score: f32, sources_count: usize } => tracing::Level::DEBUG,
|
||||
|
||||
#[error("ML detection bridge started")]
|
||||
MlBridgeStarted => tracing::Level::INFO,
|
||||
|
||||
@ -38,5 +38,11 @@ loggable! {
|
||||
|
||||
#[error("Traffic logger channel disconnected")]
|
||||
TrafficLogChannelDisconnected => tracing::Level::WARN,
|
||||
|
||||
#[error("ML circuit breaker OPEN: {failures} failures in {window_secs}s, inference disabled until reset")]
|
||||
CircuitBreakerOpen { failures: u32, window_secs: u64 } => tracing::Level::ERROR,
|
||||
|
||||
#[error("ML circuit breaker RESET: inference re-enabled after {cooldown_secs}s cooldown")]
|
||||
CircuitBreakerReset { cooldown_secs: u64 } => tracing::Level::WARN,
|
||||
}
|
||||
}
|
||||
|
||||
@ -69,8 +69,8 @@ loggable! {
|
||||
#[error("TTL sweep: {removed} blocks removed, {skipped} kept (manual ACL conflict)")]
|
||||
TtlSweepComplete { removed: u32, skipped: u32 } => tracing::Level::DEBUG,
|
||||
|
||||
#[error("SOAR log action [{level}]: threat from {source_ip} — {attack_type} (confidence: {confidence})")]
|
||||
ActionLog { level: String, source_ip: String, attack_type: String, confidence: String } => tracing::Level::INFO,
|
||||
#[error("SOAR log action [{level}]: threat from {source_ip} — {attack_type} (confidence: {confidence}, ae: {ae_score}, anomaly: {anomaly_score}, c2: {c2_score})")]
|
||||
ActionLog { level: String, source_ip: String, attack_type: String, confidence: String, ae_score: String, anomaly_score: String, c2_score: String } => tracing::Level::INFO,
|
||||
|
||||
#[error("SOAR cooldown cleanup: {removed} expired entries removed")]
|
||||
CooldownCleanup { removed: u32 } => tracing::Level::DEBUG,
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user