diff --git a/.gitignore b/.gitignore index e2fd9ba..8ee9506 100644 --- a/.gitignore +++ b/.gitignore @@ -19,4 +19,3 @@ node_modules/ *.rules *.db -net-guardia/static/rules/*.txt diff --git a/.research/state.toml b/.research/state.toml index 40f0e86..a1f31e4 100644 --- a/.research/state.toml +++ b/.research/state.toml @@ -1,13 +1,13 @@ # NetGuardia Research State -# Updated: 2026-05-19 +# Updated: 2026-05-21 [[epics]] id = "nids-v1" title = "NetGuardia NIDS v1 — Research Prototype" status = "active" description = """ -End-to-end NIDS combining eBPF packet capture, LSTM autoencoder ML inference, -Suricata-compatible rule engine, and a Next.js monitoring frontend. +End-to-end NIDS combining eBPF/AF_XDP packet capture, LSTM autoencoder ML inference, +Suricata daemon integration (49 000+ ET Open rules), and a Next.js monitoring frontend. Target: demonstrate joint ML+Rule detection superiority over single-method baselines. """ @@ -15,22 +15,60 @@ Target: demonstrate joint ML+Rule detection superiority over single-method basel id = "backend-detection" epic = "nids-v1" status = "active" -description = "ML pipeline, rule engine, fusion layer, and alert broadcast." +description = """ +ML pipeline, Suricata daemon mode, fusion layer, and alert broadcast. +Rule engine (vectorscan) fully replaced by Suricata 8 daemon + veth mirror. +Fusion engine correlates ML anomaly scores with Suricata EVE JSON alerts. +Next: TLS/JA3 analysis, XDP auto-block active response. +""" [[themes]] id = "backend-infra" epic = "nids-v1" status = "active" -description = "GeoIP, logging, eBPF statistics, HTTP API, WebSocket delivery." +description = """ +GeoIP, logging, eBPF statistics, HTTP API, WebSocket delivery. +Fixed: GeoIP thread explosion (semaphore), maxminddb debug flood. +Pending: account system (SQLite app.db), full REST API with auth. +""" [[themes]] id = "frontend" epic = "nids-v1" status = "active" -description = "Next.js dashboard, detection page, WebSocket provider, config alignment." +description = """ +Next.js dashboard, detection page, WebSocket provider, config alignment. +Aligned with UnifiedAlert API (source/severity as lowercase strings). +Open: dashboard counters never reset (accumulate forever). +""" + +[[themes]] +id = "tls-analysis" +epic = "nids-v1" +status = "planned" +description = """ +Encrypted traffic analysis without decryption. +Suricata EVE tls events -> JA3 hash whitelist + cert anomaly rules + beacon detection. +No ML model required initially; extend with classifier if FP rate is too high. +Addresses the blind spot where Suricata rules cannot inspect TLS payload. +""" + +[[themes]] +id = "active-response" +epic = "nids-v1" +status = "planned" +description = """ +XDP-layer automatic IP blocking triggered by high-confidence FusionEngine alerts. +BPF_MAP_TYPE_LRU_HASH in ingress XDP program; userspace inserts src_ip on alert. +TTL-based expiry via background tokio task. +Demonstrates sub-microsecond response latency (key academic differentiator). +""" [[themes]] id = "account-system" epic = "nids-v1" status = "parked" -description = "SQLite-backed accounts, sessions, persistent whitelist/blacklist." \ No newline at end of file +description = """ +SQLite-backed accounts, sessions, persistent whitelist/blacklist. +Deprioritised; not required for research evaluation. +""" diff --git a/TODO b/TODO index 21b9705..ea1a7be 100644 --- a/TODO +++ b/TODO @@ -1,237 +1,83 @@ -1. .csv 一開始文件格式為 traffic-yyyy-oo-zz.csv, 換天後直接變日期創新檔，例如今天啟動今天為 traffic-2026-05-11.csv 隔天變成 traffic-2026-05-12.csv [x] -2. will change tract-onnx engine to ort-tract engine [x] -3. 改善推論效能及速度，在大量資料時 [x] -4. 實現 ML + RULE 的共用 HashMap 實現共同決策結果 [x] -5. 代碼最佳化，檢查是否除了 build.rs 以外有無 .unwarp() and eprintln() [x] -6. 完成前端 detection 頁面 [x] -7. 使用 sqllite 實現帳號系統、白黑名單永久記錄 - -[x] -Suricata -提升偵測準確率： -pcre 很多規則依賴正則，沒有會漏掉很多威脅 -flow:established 減少掃握手封包的無意義處理 -Flowbits 複雜攻擊鏈的偵測（例如先偵測掃描再偵測滲透） - -減少假陽性： -$HOME_NET / $EXTERNAL_NET 區分內外網，很多規則依賴這個 -flow:established 順便過濾掉不完整連線 - ============================================================ -PRIORITY BACKLOG (updated 2026-05-19) +NetGuardia TODO (updated 2026-05-21) ============================================================ --- MUST DO (system is not usable without these) ----------- +-- DONE (archived) ---------------------------------------- -[x] byte_test / byte_jump / byte_extract - Completed 2026-05-15. - build.rs: parse_byte_test/jump/extract -> SQLite byte_ops table. - rule_engine.rs: eval_byte_ops() -> eval_byte_test/jump, read_bytes(). - Supports: relative, endian, string mode (dec/hex/oct), bitmask, vars. +[x] CSV rolling log with date-based filenames +[x] Migrate inference engine to ort-tract (pure Rust) +[x] Improve ML inference throughput under high load +[x] ML + Rule fusion decision layer (FusionEngine) +[x] Code audit: remove .unwrap() / eprintln() outside build.rs +[x] Frontend detection page +[x] GeoIP unbounded thread explosion (semaphore cap = 8) +[x] maxminddb DEBUG log flood (EnvFilter "maxminddb=warn") +[x] Frontend UnifiedAlert API alignment (detection.tsx, WebSocketProvider, config.ts, dashboard.tsx) +[x] Rule engine -> replaced entirely by Suricata daemon mode + Removed: vectorscan, rusqlite, protolens, pcre2, byte_test/jump/extract, + app-layer-protocol, threshold, QUIC parser, suppress list (SQLite). + Replaced by: suricata daemon + veth mirror + EVE JSON unix socket. + Suricata handles all signature matching; NetGuardia reads alerts via output.rs. +[x] Suricata EVE socket race condition (bind before spawn) +[x] af-packet block-size too small (32768 -> 131072) +[x] Suricata noisy rule categories (exclude emerging-info/policy/user_agents) +[x] suppress.conf: add known-benign SIDs (2013504 APT User-Agent etc.) -[x] app-layer-protocol: keyword - Completed 2026-05-15. - build.rs: extract_alproto() parses keyword -> (proto_id, negated). - rule_engine.rs: pkt.proto as u8 == sig.alproto filter before chain. - Protocols mapped: http(1) http2(2) tls(3) dns(4) ssh(5) smtp(6) - ftp(7) mqtt(8) quic(9). +-- SHOULD DO (detection capability) ---------------------- -[x] ML + Rule fusion decision layer - Core differentiator of the project. Currently ML and rule - engine produce independent alerts with no cross-awareness. - Design: shared per-flow HashMap; two fusion modes: - AND - alert only when both agree - OR - alert when either fires (with source tag) - Files: detection/ml/engine.rs, detection/rule/rule_engine.rs, - new detection/fusion.rs, model/ml_detection.rs +[ ] TLS / JA3 fingerprint analysis + Goal: detect encrypted malware C2 / beaconing without decryption. + Step 1: add tls event type to suricata.yaml EVE outputs (extended: yes). + Step 2: parse event_type == "tls" in output.rs -> extract ja3.hash, sni, cert info. + Step 3: build JA3 whitelist (HashSet) from observed normal traffic baseline. + Step 4: alert on unknown JA3 hash + check cert anomalies (self-signed, expired, CN mismatch). + Step 5: beacon detection -- low stddev of inter-connection interval to same dst_ip on 443. + No model needed for steps 1-5; add dedicated classifier only if false-positive rate + is unacceptable after baseline calibration. + Files: detection/suricata/output.rs, detection/suricata/tls.rs (new), + model/log/suricata.rs, model/error/suricata.rs -[x] threshold: complete three modes - Completed 2026-05-17. - build.rs: extract_threshold() parses type/track/count/seconds -> threshold_entries table. - rule_engine.rs: ThresholdKey (BySrc/ByDst/ByBoth/ByRule); ThresholdState; - parking_lot::Mutex>; should_alert() mirrors - Suricata ThresholdCheckUpdate() exactly: - threshold (3) - fire on every Nth hit; reset after fire; no alert on window expiry - limit (1) - fire on first N hits per window; alert after window reset - both (2) - fire exactly on Nth hit (==, not >=); silence for N+1...; fire on expiry if count==1 - Track: TRACK_DST=1 TRACK_SRC=2 TRACK_RULE=3 TRACK_BOTH=5 TRACK_FLOW=6 (matches Suricata header) +[ ] XDP active response (auto-block) + Goal: block confirmed attacker IPs at kernel level (< 1 us per packet). + Design: BPF_MAP_TYPE_LRU_HASH keyed by src IPv4/IPv6; XDP program checks map + and returns XDP_DROP before any userspace processing. + Trigger: FusionEngine fires a fusion or high-confidence ML alert -> insert src_ip + into block map via userspace BPF map update API. + Expiry: separate tokio task sweeps entries older than block_ttl_secs (config). + API: POST /api/blacklist still works for manual entries; auto-block is additive. + Files: ingress-ebpf/src/main.rs (map lookup + XDP_DROP), + core/ebpf/xsk_manager.rs (map fd plumbing), + detection/fusion.rs (trigger auto-block on high-confidence events) -- SHOULD DO (runtime management) ------------------------ -[ ] Account system + persistent list DB (updated 2026-05-18) - Two SQLite files, separated by concern: - - static/db/rules.db (rule-related, already exists) - suppress (id, sid INTEGER, track INTEGER, ip_net TEXT, comment TEXT, created_at) - - track: 1=by_src 2=by_dst 4=by_either (Suricata values) - - already populated by build.rs; API writes go here; RuleEngine reloads on change - - static/db/app.db (account + network list, new) - accounts (id, username, password_hash, role, created_at) - - role: "admin" | "viewer" - - password_hash: argon2 or bcrypt - sessions (token TEXT PK, account_id, expires_at) - - token: 32-byte random hex, expires in 24 h - whitelist (id, ip_net TEXT, comment TEXT, created_at) - - packets from whitelisted CIDRs skip rule + ML evaluation entirely - blacklist (id, ip_net TEXT, comment TEXT, action TEXT, created_at) - - action: "alert" | "drop" - - packets from blacklisted CIDRs auto-alert without rule evaluation +[ ] Account system + persistent lists + Two SQLite files: + static/db/app.db + accounts (id, username, password_hash, role, created_at) + sessions (token TEXT PK, account_id, expires_at) + whitelist (id, ip_net TEXT, comment TEXT, created_at) + blacklist (id, ip_net TEXT, action TEXT, comment TEXT, created_at) Migration: CREATE TABLE IF NOT EXISTS at startup in AppServices::init(). File: core/infrastructure/app_db.rs (new) -[ ] HTTP API - accounts / suppress / whitelist / blacklist / rules - All routes require session token in Authorization: Bearer header - except POST /api/auth/login. - - Auth: - POST /api/auth/login - { username, password } -> { token, expires_at } - POST /api/auth/logout - invalidate current token - GET /api/auth/me - current account info - - Accounts (admin only): - GET /api/accounts - list accounts - POST /api/accounts - create account { username, password, role } - PUT /api/accounts/:id/password - change password - DELETE /api/accounts/:id - delete account - - Suppress: - GET /api/suppress - list all entries - POST /api/suppress - add { sid, track, ip_net, comment } - DELETE /api/suppress/:id - remove entry - (runtime effect: RuleEngine reloads suppress table after write) - - Whitelist: - GET /api/whitelist - list entries - POST /api/whitelist - add { ip_net, comment } - DELETE /api/whitelist/:id - remove - - Blacklist: - GET /api/blacklist - list entries - POST /api/blacklist - add { ip_net, action, comment } - DELETE /api/blacklist/:id - remove - - Rules: - GET /api/rules - list loaded rules (sid, msg, enabled) - POST /api/rules/reload - trigger hot-reload - - System: - POST /api/system/restart - graceful restart - - Files: web/routes/auth.rs, web/routes/accounts.rs, web/routes/lists.rs, - web/routes/rules.rs, web/routes/system.rs - Middleware: web/middleware/auth.rs (token extractor + role check) - -[ ] Hot-reload: rules + suppress without process restart - Trigger: POST /api/rules/reload OR SIGHUP signal. - Steps: - 1. build.rs logic extracted into a runtime fn rebuild_rules_db(rules_dir) -> Result<()> - that re-parses *.rules files and rewrites rules.db in a temp path. - 2. RuleEngine::reload(new_db_path) acquires a write lock, swaps BlockDatabase + - sigs vec + SuppressList atomically; old engine dropped after swap. - 3. ML engine is NOT restarted (model/config unchanged). - 4. In-flight threshold states are preserved across reload to avoid counter reset. - Files: detection/rule/rule_engine.rs (add reload()), build.rs (extract parse logic), - core/infrastructure/app_services.rs (wire SIGHUP handler via tokio::signal). +[ ] HTTP API -- auth / accounts / lists / system + All routes require Bearer token except POST /api/auth/login. + Auth: POST /api/auth/login, POST /api/auth/logout, GET /api/auth/me + Accounts: GET/POST /api/accounts, PUT /api/accounts/:id/password, DELETE /api/accounts/:id + Suppress: GET/POST/DELETE /api/suppress (writes suppress.conf + signals Suricata reload) + Lists: GET/POST/DELETE /api/whitelist, /api/blacklist + System: POST /api/system/restart + Files: web/routes/{auth,accounts,lists,system}.rs, web/middleware/auth.rs [ ] Graceful restart - Trigger: POST /api/system/restart OR SIGTERM + re-exec. - Steps: - 1. Drain in-flight packets: wait up to 2 s for flow tracker to flush. - 2. Detach eBPF programs (XDP unload) cleanly. - 3. Flush CSV writer and close traffic log. - 4. tokio runtime shutdown, then re-exec self via std::process::Command. - Why re-exec instead of just reload: eBPF object files must be re-loaded from disk - when kernel map layouts change; a hot-reload cannot cover this case. - Files: core/ebpf/ (detach helpers), main.rs (signal handler + re-exec path). + Trigger: POST /api/system/restart or SIGTERM. + Steps: drain flow tracker (2 s), XDP unload, flush CSV, tokio shutdown, re-exec. + Files: core/ebpf/ (detach helpers), main.rs (signal handler + re-exec) --- SHOULD DO (usable but blind spots remain) -------------- +-- OPEN QUESTIONS ----------------------------------------- -[x] QUIC parser - Completed 2026-05-16. Full port of Suricata rust/src/quic/parser.rs - and rust/src/quic/frames.rs. - RFC QUIC v1/v2: HKDF key derivation + AES-128-GCM Initial decryption - + TLS ClientHello SNI extraction from CRYPTO frames. - gQUIC Q043-Q046: plaintext Initial frames; SNI from CHLO tag-value - structure (StreamTag::Sni 0x534e4900) in STREAM frames (0x80+). - File: app_layer/quic.rs; integrated in mod.rs + rule_engine.rs. - -[x] suppress list - Completed 2026-05-15. - Suricata suppress.conf format: gen_id N, sig_id N, track by_src|by_dst, ip addr. - File: detection/rule/suppress.rs; loaded at startup; filters results - in rule_engine.rs after matching (results.retain(|m| !suppress.is_suppressed(...))). - Placeholder: static/db/suppress.conf (empty, add entries as needed). - -[x] isdataat - Completed 2026-05-15 (included with byte_test/jump/extract). - ByteOp kind=3; supports negated form (!isdataat) and relative flag. - build.rs: parse_isdataat(); rule_engine.rs: eval_byte_ops() case 3. - --- INFRASTRUCTURE FIXES (2026-05-19) ---------------------- - -[x] GeoIP unbounded thread explosion - Completed 2026-05-19. - Root cause: statistics.rs calls join_all() across all flows per WebSocket push; - each cache-miss triggers spawn_blocking with no concurrency cap. Observed - 60+ simultaneous OS threads (ThreadId 83-148) under normal traffic. - Fix: added Arc(MAX_CONCURRENT_DB_LOOKUPS=8) to GeoIpService. - acquire_owned() before spawn_blocking; OwnedSemaphorePermit moved into - closure and held until blocking call returns. Cache hits and private IP - paths bypass semaphore. After fix: max 8 concurrent threads at any time. - File: core/infrastructure/geoip.rs - -[x] maxminddb DEBUG log flood in development builds - Completed 2026-05-19. - Root cause: logging.rs sets global tracing level to Level::DEBUG when - cfg!(debug_assertions) (non-release builds). maxminddb emits ~50 debug - lines per IP lookup via tracing::debug!() in its deserializer. - Fix: added EnvFilter directive "maxminddb=warn" which takes precedence - over the global level. Release builds are unaffected (already INFO). - File: utils/logging.rs - --- FRONTEND ALIGNMENT (2026-05-19) ------------------------ - -[x] Frontend detection page — UnifiedAlert API alignment - Completed 2026-05-19. - Old detection.tsx used AlertLog type with rf_score/ensemble_score and - confidence-based severity (4 levels). Backend sends UnifiedAlert with - source (ml|rule|fusion), severity (high|critical), rule_sid, rule_msg, - and nullable attack_type. All serde enum variants are snake_case lowercase. - Files updated: .research/frontend-refactor/detection.tsx - Changes: new UnifiedAlert type inline, SOURCE_CONFIG/SEVERITY_CONFIG with - lowercase keys, dual filter (severity x source), 7 stats cards, - ML scores panel only for ml/fusion, rule details panel only for rule/fusion, - isPausedRef pattern to prevent WS re-subscription on pause toggle. - -[x] Frontend config URL mismatches - Completed 2026-05-19. - Six URL mismatches corrected; port updated to 2048: - /control/access_control/... -> /ebpf/access_control/... (x4) - /statistics/websocket/... -> /ebpf/statistics/websocket/... (x2) - /health/websocket/system_health -> /health/websocket/metrics - websocketUrl.mlAlert (/ml/websocket/alert) -> detectionAlert (/detection/websocket/alert) - File: .research/frontend-refactor/config.ts - -[x] WebSocketProvider — naming and Subject upgrade - Completed 2026-05-19. - getAiAlertStream -> getDetectionAlertStream throughout interface, context - default, useCallback, and initializeWebSockets. Subject -> BehaviorSubject - so late subscribers receive the last cached value immediately. Added - getLatestFlowData() and getLatestSystemHealth() sync cache accessors. - File: .research/frontend-refactor/WebSocketProvider.tsx - -[x] Dashboard page — detection stream alignment - Completed 2026-05-19. - Removed unused AlertLog import. getAiAlertStream -> getDetectionAlertStream. - attack_type null guard added before updating attackTypeCounts. - Locale zh-TW -> en-GB for chart axis time labels. - File: .research/frontend-refactor/dashboard.tsx - --- OPEN QUESTIONS (not blocking, worth revisiting) -------- -[ ] attackTypeCounts / protocolCounts in dashboard never reset - Accumulate for the entire session lifetime. After hours of runtime, the - donut charts show historical totals rather than recent activity. - Consider: sliding window reset (e.g., clear counts every 1 hour) or - limit to last N alerts. - File: .research/frontend-refactor/dashboard.tsx \ No newline at end of file +[ ] Dashboard attack/protocol counters never reset + Accumulate for entire session lifetime; charts show historical totals. + Consider: sliding window reset every N minutes, or cap at last 500 alerts. + File: net-guardia-frontend/...dashboard.tsx diff --git a/net-guardia/static/rules/info.txt b/net-guardia/static/rules/info.txt index 3d14aa6..6132249 100644 --- a/net-guardia/static/rules/info.txt +++ b/net-guardia/static/rules/info.txt @@ -1 +1 @@ -// this's open rule folder, so you can download the open rule from https://rules.emergingthreats.net/open/suricata-8.0/emerging.rules.tar.gz +// this's open rule folder, so you can download the open rule from https://rules.emergingthreats.net/open/suricata-8.0/emerging.rules.tar.gz \ No newline at end of file