diff --git a/backend/db.py b/backend/db.py index 2ab5570..11004ee 100644 --- a/backend/db.py +++ b/backend/db.py @@ -48,6 +48,15 @@ class IngestSnapshot(Base): recorded_at = Column(DateTime, default=datetime.utcnow) +class RuleFiringCache(Base): + __tablename__ = "rule_firing_cache" + id = Column(Integer, primary_key=True) + rule_name = Column(String, unique=True, index=True) + alert_count = Column(Integer, default=0) + period_days = Column(Integer, default=30) + checked_at = Column(DateTime, default=datetime.utcnow) + + def get_db(): db = SessionLocal() try: diff --git a/backend/main.py b/backend/main.py index 32cf7da..762fc87 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,6 +1,6 @@ from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from db import engine, Base, get_db, ParsedRule +from db import engine, Base, get_db, ParsedRule, RuleFiringCache from routers import coverage, ingest, settings, quality Base.metadata.create_all(bind=engine) @@ -14,6 +14,15 @@ with engine.connect() as _conn: _conn.execute(text( "ALTER TABLE active_sources ADD COLUMN IF NOT EXISTS unlabelled BOOLEAN DEFAULT FALSE" )) + _conn.execute(text( + "CREATE TABLE IF NOT EXISTS rule_firing_cache (" + "id SERIAL PRIMARY KEY, " + "rule_name VARCHAR UNIQUE, " + "alert_count INTEGER DEFAULT 0, " + "period_days INTEGER DEFAULT 30, " + "checked_at TIMESTAMP" + ")" + )) _conn.commit() app = FastAPI(title="SIEM Toolkit", version="1.0.0") diff --git a/backend/routers/coverage.py b/backend/routers/coverage.py index 22c635d..92e746c 100644 --- a/backend/routers/coverage.py +++ b/backend/routers/coverage.py @@ -4,7 +4,7 @@ from fastapi import APIRouter, UploadFile, File, Depends, HTTPException from pydantic import BaseModel from sqlalchemy.orm import Session from datetime import datetime -from db import get_db, ParsedRule, ParserField, ActiveSource +from db import get_db, ParsedRule, ParserField, ActiveSource, RuleFiringCache from services import s1_client, rule_parser DETECTIONS_FILE = os.environ.get("DETECTIONS_FILE", "/app/data/detections.json") @@ -12,6 +12,63 @@ DETECTIONS_FILE = os.environ.get("DETECTIONS_FILE", "/app/data/detections.json") router = APIRouter() +def _extract_mitre(rule: dict) -> tuple[list[str], list[dict]]: + """Extract (tactics, techniques) from a raw S1 rule dict. + Handles multiple field name conventions across S1 API versions.""" + tactics: list[str] = [] + techniques: list[dict] = [] + + for key in ("tactic", "tactics", "mitreTactic", "mitreTactics", "attack.tactic"): + val = rule.get(key) + if isinstance(val, str) and val: + tactics.extend(v.strip() for v in val.split(",") if v.strip()) + elif isinstance(val, list): + for v in val: + if isinstance(v, str) and v: + tactics.append(v.strip()) + elif isinstance(v, dict): + n = v.get("name") or v.get("tactic") or "" + if n: + tactics.append(n.strip()) + + for key in ("technique", "techniques", "mitreTechnique", "mitreTechniques", + "attack.technique", "mitreAttack"): + val = rule.get(key) + if isinstance(val, str) and val: + for part in val.split(","): + part = part.strip() + if not part: + continue + if part.startswith("T") and len(part) >= 2 and part[1:5].replace(".", "").isdigit(): + tid, _, tname = part.partition(" - ") + techniques.append({"id": tid.strip(), "name": tname.strip() or tid.strip()}) + else: + techniques.append({"id": "", "name": part}) + elif isinstance(val, list): + for v in val: + if isinstance(v, str) and v.strip(): + part = v.strip() + if part.startswith("T") and len(part) >= 5 and part[1:5].replace(".", "").isdigit(): + techniques.append({"id": part, "name": part}) + else: + techniques.append({"id": "", "name": part}) + elif isinstance(v, dict): + tid = v.get("id") or v.get("techniqueId") or v.get("technique_id") or "" + tname = v.get("name") or v.get("technique") or v.get("techniqueName") or tid + if tid or tname: + techniques.append({"id": str(tid).strip(), "name": str(tname).strip()}) + + seen_ids: set = set() + unique_techniques = [] + for t in techniques: + key_t = t["id"] or t["name"] + if key_t not in seen_ids: + seen_ids.add(key_t) + unique_techniques.append(t) + + return list(dict.fromkeys(tactics)), unique_techniques + + def _star_query_texts(rule: dict) -> list[str]: """ Extract all PowerQuery/filter strings from a STAR rule. @@ -94,12 +151,17 @@ def _import_from_api_rules(db, rules: list) -> int: seen_ids.add(rule_id) sources = rule.get("sources") or [] + tactics, techniques = _extract_mitre(rule) db.add(ParsedRule( rule_id=rule_id, name=rule.get("name", "unnamed"), rule_type="library", fields_used=[], # API rules don't expose field-level info - raw=json.dumps({"data_sources": sources}), + raw=json.dumps({ + "data_sources": sources, + "tactics": tactics, + "techniques": techniques, + }), )) loaded += 1 if loaded % 500 == 0: @@ -142,12 +204,17 @@ def _import_detections(db, detections_file: str) -> int: continue seen_ids.add(rule_id) + tactics, techniques = _extract_mitre(rule) db.add(ParsedRule( rule_id=rule_id, name=rule.get("name", "unnamed"), rule_type="library", fields_used=list(all_fields), - raw=json.dumps({"data_sources": list(set(data_sources))}), + raw=json.dumps({ + "data_sources": list(set(data_sources)), + "tactics": tactics, + "techniques": techniques, + }), )) loaded += 1 if loaded % 500 == 0: @@ -561,6 +628,12 @@ def get_coverage_map(db: Session = Depends(get_db)): parser_fields_rows = db.query(ParserField).all() rules = db.query(ParsedRule).all() + firing_cache: dict[str, int] = { + row.rule_name: row.alert_count + for row in db.query(RuleFiringCache).all() + } + firing_cache_populated = len(firing_cache) > 0 + # parser_name → set of field names (for field count display) parser_index: dict[str, set] = {} for pf in parser_fields_rows: @@ -675,7 +748,11 @@ def get_coverage_map(db: Session = Depends(get_db)): else: needed_count += 1 # stub_parser and parser_needed both count as needing work - rules_for_src: list = [r for r in rule_by_source.get(src.source_name, []) if r["type"] == "library"] + rules_for_src: list = [ + {**r, "alert_count": firing_cache.get(r["rule"], 0)} + for r in rule_by_source.get(src.source_name, []) + if r["type"] == "library" + ] # Close-match suggestions — shown when there are no library rules for this source. close_matches: list = [] @@ -779,6 +856,7 @@ def get_coverage_map(db: Session = Depends(get_db)): "unlabelled_events": _unlabelled_event_count, "parsers_loaded": len(parser_index), "rules_loaded": len(rules), + "firing_cache_populated": firing_cache_populated, }, "sources": sources_out, "synced_at": synced_at, @@ -794,6 +872,175 @@ def get_stub_parsers(): return {"stubs": stubs, "count": len(stubs)} +@router.get("/mitre") +def get_mitre_coverage(db: Session = Depends(get_db)): + rules = db.query(ParsedRule).filter_by(rule_type="library").all() + + TACTIC_ORDER = [ + "Reconnaissance", "Resource Development", "Initial Access", "Execution", + "Persistence", "Privilege Escalation", "Defense Evasion", "Credential Access", + "Discovery", "Lateral Movement", "Collection", "Command and Control", + "Exfiltration", "Impact", "Uncategorized", + ] + + tactic_map: dict[str, dict] = {} + no_mitre_count = 0 + + for rule in rules: + try: + raw_data = json.loads(rule.raw) if rule.raw else {} + except Exception: + raw_data = {} + tactics = raw_data.get("tactics", []) + techniques = raw_data.get("techniques", []) + if not tactics and not techniques: + no_mitre_count += 1 + continue + if not tactics: + tactics = ["Uncategorized"] + for tactic in tactics: + if tactic not in tactic_map: + tactic_map[tactic] = {"techniques": {}, "rule_count": 0} + tactic_map[tactic]["rule_count"] += 1 + for tech in techniques: + key_t = tech["id"] or tech["name"] + if key_t: + tactic_map[tactic]["techniques"][key_t] = tech["name"] or key_t + + def _tactic_sort_key(name: str) -> int: + try: + return TACTIC_ORDER.index(name) + except ValueError: + return len(TACTIC_ORDER) + + tactics_out = [] + total_techniques = 0 + for tactic_name in sorted(tactic_map.keys(), key=_tactic_sort_key): + tech_dict = tactic_map[tactic_name]["techniques"] + techniques_list = [ + {"id": k if (k.startswith("T") and len(k) >= 4) else "", "name": v} + for k, v in sorted(tech_dict.items()) + ] + total_techniques += len(techniques_list) + tactics_out.append({ + "tactic": tactic_name, + "rule_count": tactic_map[tactic_name]["rule_count"], + "technique_count": len(techniques_list), + "techniques": techniques_list, + }) + + return { + "tactics": tactics_out, + "total_rules": len(rules), + "rules_with_mitre": len(rules) - no_mitre_count, + "rules_without_mitre": no_mitre_count, + "total_techniques": total_techniques, + "tactic_count": len(tactics_out), + } + + +@router.post("/sync-rule-firing") +async def sync_rule_firing(period_days: int = 30, db: Session = Depends(get_db)): + """Query SDL for alert/threat counts by rule name over the last N days. + Tries multiple field name patterns until one returns results. + Caches results in rule_firing_cache table.""" + from datetime import datetime, timedelta + now = datetime.utcnow() + from_dt = (now - timedelta(days=period_days)).strftime("%Y-%m-%dT%H:%M:%S.000Z") + to_dt = now.strftime("%Y-%m-%dT%H:%M:%S.000Z") + + FIRING_QUERIES = [ + ("| filter ruleName != '' | group alerts=count() by ruleName | sort -alerts | limit 2000", "ruleName"), + ("| filter threatInfo.detectionEngineRule.name != '' | group alerts=count() by threatInfo.detectionEngineRule.name | sort -alerts | limit 2000", "threatInfo.detectionEngineRule.name"), + ("| filter alert.ruleName != '' | group alerts=count() by alert.ruleName | sort -alerts | limit 2000", "alert.ruleName"), + ] + + result_rows = [] + query_used = None + errors = [] + + for query, name_field in FIRING_QUERIES: + try: + result = await s1_client.run_powerquery(query, from_dt, to_dt, max_count=10_000_000) + err = result.get("error") if isinstance(result, dict) else None + if err: + errors.append(f"{name_field}: {err}") + continue + rows = result.get("events", []) + if rows: + # Remap the name field to a standard key + result_rows = [{"rule_name": r.get(name_field, r.get("ruleName", "")), "alerts": r.get("alerts", 0)} for r in rows] + result_rows = [r for r in result_rows if r["rule_name"]] + if result_rows: + query_used = query + break + except Exception as e: + errors.append(f"{name_field}: {e}") + continue + + if not result_rows: + return { + "synced": 0, + "period_days": period_days, + "rules_with_alerts": 0, + "query_used": None, + "message": "No alert data found. Errors: " + "; ".join(errors) if errors else "No alert data found — SDL may not have alert events in this time window.", + } + + # Upsert into cache + checked_at = datetime.utcnow() + for row in result_rows: + existing = db.query(RuleFiringCache).filter_by(rule_name=row["rule_name"]).first() + if existing: + existing.alert_count = row["alerts"] + existing.period_days = period_days + existing.checked_at = checked_at + else: + db.add(RuleFiringCache( + rule_name=row["rule_name"], + alert_count=row["alerts"], + period_days=period_days, + checked_at=checked_at, + )) + db.commit() + + return { + "synced": len(result_rows), + "period_days": period_days, + "rules_with_alerts": len(result_rows), + "query_used": query_used, + } + + +@router.get("/rule-firing-cache") +def get_rule_firing_cache(db: Session = Depends(get_db)): + """Return all cached rule firing data sorted by alert count descending.""" + rows = db.query(RuleFiringCache).order_by(RuleFiringCache.alert_count.desc()).all() + total_rules = db.query(ParsedRule).filter_by(rule_type="library").count() + fired = [r for r in rows if r.alert_count > 0] + never_fired_count = total_rules - len(fired) + period_days = rows[0].period_days if rows else 30 + checked_at = rows[0].checked_at.isoformat() if rows and rows[0].checked_at else None + return { + "rules": [ + { + "rule_name": r.rule_name, + "alert_count": r.alert_count, + "period_days": r.period_days, + "checked_at": r.checked_at.isoformat() if r.checked_at else None, + } + for r in rows + ], + "summary": { + "rules_monitored": len(rows), + "fired_in_period": len(fired), + "never_fired": never_fired_count, + "period_days": period_days, + "checked_at": checked_at, + }, + } + + @router.delete("/reset") def reset_data(db: Session = Depends(get_db)): db.query(ParsedRule).delete() diff --git a/frontend/index.html b/frontend/index.html index 7fd6010..ecdd6dd 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -22,6 +22,7 @@ Ingest Dashboard Parser Quality Onboarding + Threat Coverage
@@ -559,7 +560,19 @@ function cvSetFilter(f) { } function detectionsCell(s) { + const firingPopulated = cvData?.summary?.firing_cache_populated === true if (s.rule_count) { + if (firingPopulated) { + const ruleItems = (s.rules || []).map(r => { + const alerts = r.alert_count || 0 + if (alerts > 0) { + return `${esc(r.rule.length > 40 ? r.rule.slice(0, 40) + '…' : r.rule)} (${alerts})` + } else { + return `⚠ ${esc(r.rule.length > 40 ? r.rule.slice(0, 40) + '…' : r.rule)}` + } + }) + return `
${ruleItems.join('')}
` + } return `${s.rule_count} rule${s.rule_count !== 1 ? 's' : ''}` } if (s.close_matches && s.close_matches.length) { @@ -1394,6 +1407,222 @@ async function qtTest() { } finally { setBtn('btn-qt', false, 'Test') } } +// ── Threat Coverage ─────────────────────────────────────────────────────── + +function renderThreat() { + set(`
+
+

Threat Coverage

+

MITRE ATT&CK heatmap and detection rule firing status

+
+ + +
+
+
+

MITRE ATT&CK Coverage

+

Detection rules mapped to ATT&CK tactics and techniques

+
+ +
+
+
+
+
Loading MITRE coverage…
+
+
+ + +
+
+
+

Rule Firing Status

+

Rules that have never triggered an alert may be misconfigured or require data sources not yet active.

+
+
+ + +
+
+
+
+
+
+
`) + loadMitre() + loadFiringStatus() +} + +async function syncLibraryThreat() { + setBtn('btn-sync-library-threat', true) + const errEl = document.getElementById('mitre-err') + if (errEl) errEl.innerHTML = '' + try { + const r = await apiPost('/api/coverage/load-detections', {}) + if (errEl) { + errEl.innerHTML = `
✓ ${r.loaded} detection rules synced from ${r.source === 'api' ? 'S1 API' : 'local file'}
` + setTimeout(() => { errEl.innerHTML = '' }, 4000) + } + loadMitre() + } catch(e) { + if (errEl) errEl.innerHTML = errBox(e.message) + } finally { setBtn('btn-sync-library-threat', false, 'Sync Detection Library') } +} + +async function loadMitre() { + const gridEl = document.getElementById('mitre-grid') + const statsEl = document.getElementById('mitre-stats') + if (!gridEl) return + gridEl.innerHTML = '
Loading MITRE coverage…
' + try { + const data = await apiGet('/api/coverage/mitre') + + // Stats row + if (statsEl) { + statsEl.innerHTML = ` + ${statCard('Total Library Rules', data.total_rules, 'text-gray-200')} + ${statCard('Rules with MITRE Mapping', data.rules_with_mitre, 'text-purple-400')} + ${statCard('Tactics Covered', data.tactic_count, 'text-blue-400')} + ${statCard('Techniques Covered', data.total_techniques, 'text-emerald-400')}` + } + + if (!data.tactics || data.tactics.length === 0) { + gridEl.innerHTML = `
+ No MITRE data found. Sync the Detection Library to populate MITRE mappings. +
` + return + } + + gridEl.innerHTML = data.tactics.map(t => { + const borderColor = t.rule_count >= 20 ? 'border-l-purple-500' : t.rule_count >= 5 ? 'border-l-blue-500' : 'border-l-slate-600' + const badgeColor = t.rule_count >= 20 ? 'bg-purple-900/60 text-purple-300 border-purple-700' : t.rule_count >= 5 ? 'bg-blue-900/60 text-blue-300 border-blue-700' : 'bg-slate-800/60 text-slate-400 border-slate-700' + + const MAX_SHOWN = 12 + const chips = t.techniques.slice(0, MAX_SHOWN).map(tech => + `${esc(tech.id || tech.name)}` + ).join('') + + const expandId = 'mitre-expand-' + t.tactic.replace(/[^a-z0-9]/gi, '_') + const moreChips = t.techniques.slice(MAX_SHOWN).map(tech => + `${esc(tech.id || tech.name)}` + ).join('') + const moreSection = t.techniques.length > MAX_SHOWN ? ` + + ` : '' + + return `
+
+ ${esc(t.tactic)} + ${t.rule_count} rule${t.rule_count !== 1 ? 's' : ''} +
+ ${t.techniques.length > 0 ? ` +
${chips}
+ ${moreSection}` : `

No technique IDs mapped

`} +
` + }).join('') + } catch(e) { + if (gridEl) gridEl.innerHTML = `
${esc(e.message)}
` + } +} + +function mitreToggleMore(id, btn) { + const el = document.getElementById(id) + if (!el) return + const hidden = el.classList.toggle('hidden') + btn.textContent = hidden ? '+' + btn.textContent.replace(/^[^ ]+\s*/, '') + ' more' : 'Show less' + // Re-calculate count when hiding + if (hidden) { + const count = el.querySelectorAll('span').length + btn.textContent = '+' + count + ' more' + } +} + +async function syncRuleFiring() { + setBtn('btn-sync-firing', true) + const errEl = document.getElementById('firing-err') + if (errEl) errEl.innerHTML = '' + try { + const period = +document.getElementById('firing-period').value || 30 + const r = await apiPost(`/api/coverage/sync-rule-firing?period_days=${period}`, {}) + if (r.message) { + if (errEl) errEl.innerHTML = `
⚠ ${esc(r.message)}
` + } else { + if (errEl) { + errEl.innerHTML = `
✓ Synced ${r.synced} rules over ${r.period_days}d
` + setTimeout(() => { errEl.innerHTML = '' }, 4000) + } + loadFiringStatus() + } + } catch(e) { + if (errEl) errEl.innerHTML = errBox(e.message) + } finally { setBtn('btn-sync-firing', false, 'Sync Alert Firing Status') } +} + +async function loadFiringStatus() { + const summaryEl = document.getElementById('firing-summary') + const tableEl = document.getElementById('firing-table') + if (!summaryEl || !tableEl) return + try { + const data = await apiGet('/api/coverage/rule-firing-cache') + const s = data.summary + + if (!data.rules || data.rules.length === 0) { + summaryEl.innerHTML = '' + tableEl.innerHTML = `
+ No firing data cached yet. Click Sync Alert Firing Status to query the SDL. +
` + return + } + + summaryEl.innerHTML = ` +
+ ${statCard('Rules Monitored', s.rules_monitored, 'text-gray-200')} + ${statCard(`Fired in ${s.period_days}d`, s.fired_in_period, 'text-emerald-400')} + ${statCard('Never Fired', s.never_fired, s.never_fired > 0 ? 'text-amber-400' : 'text-gray-500')} +
` + + const top20 = data.rules.slice(0, 20) + const rows = top20.map((r, i) => { + const badge = r.alert_count > 0 + ? `Active` + : `Silent` + return ` + ${esc(r.rule_name)} + ${r.alert_count.toLocaleString()} + ${badge} + ` + }).join('') + + tableEl.innerHTML = ` +
+ + + + + + + ${rows} +
Rule NameAlerts (${s.period_days}d)Status
+
+ ${s.checked_at ? `

Last synced: ${new Date(s.checked_at).toLocaleString()}

` : ''}` + } catch(e) { + if (tableEl) tableEl.innerHTML = `

${esc(e.message)}

` + } +} + // ── Router ──────────────────────────────────────────────────────────────── function set(html) { document.getElementById('main').innerHTML = html } @@ -1411,6 +1640,7 @@ function route() { else if (h === '#/ingest') { updateNav('ingest'); renderIngest() } else if (h === '#/quality') { updateNav('quality'); renderQuality() } else if (h === '#/onboarding') { updateNav('onboarding'); renderOnboarding() } + else if (h === '#/threat') { updateNav('threat'); renderThreat() } else if (h === '#/settings') { updateNav('settings'); renderSettings() } else { updateNav('home'); renderHome() } }