mirror of
https://github.com/marcredhat/SIEM-toolkit-patched
synced 2026-06-14 06:48:11 +00:00
Initial commit: SIEM Toolkit for SentinelOne
Dockerized SecOps toolkit with: - Coverage Map: STAR rule vs SDL parser field coverage analysis - Ingest Dashboard: PowerQuery-powered event volume and source breakdown - Onboarding Assistant: AI-guided log source onboarding with Claude - Parser management via SDL MCP integration Stack: FastAPI + PostgreSQL backend, nginx-served HTML frontend, Docker Compose. PowerQuery runs via Scalyr XDR API (SDL_XDR_URL + SDL_LOG_READ_KEY). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mick
@@ -0,0 +1,10 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"]
|
||||
@@ -0,0 +1,46 @@
|
||||
import os
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, Text
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import declarative_base, sessionmaker
|
||||
from datetime import datetime
|
||||
|
||||
DATABASE_URL = os.environ.get("DATABASE_URL", "postgresql://siem:siem@db:5432/siem")
|
||||
|
||||
engine = create_engine(DATABASE_URL)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
Base = declarative_base()
|
||||
|
||||
|
||||
class ParsedRule(Base):
|
||||
__tablename__ = "parsed_rules"
|
||||
id = Column(Integer, primary_key=True)
|
||||
rule_id = Column(String, unique=True, index=True)
|
||||
name = Column(String)
|
||||
rule_type = Column(String) # 'star' or 'sigma'
|
||||
fields_used = Column(JSONB)
|
||||
raw = Column(Text)
|
||||
cached_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
class ParserField(Base):
|
||||
__tablename__ = "parser_fields"
|
||||
id = Column(Integer, primary_key=True)
|
||||
parser_name = Column(String, index=True)
|
||||
field_name = Column(String)
|
||||
field_type = Column(String)
|
||||
|
||||
|
||||
class IngestSnapshot(Base):
|
||||
__tablename__ = "ingest_snapshots"
|
||||
id = Column(Integer, primary_key=True)
|
||||
period_days = Column(Integer)
|
||||
data = Column(JSONB)
|
||||
recorded_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,24 @@
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from db import engine, Base
|
||||
from routers import coverage, ingest
|
||||
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
app = FastAPI(title="SIEM Toolkit", version="1.0.0")
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["http://localhost:3001"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(coverage.router, prefix="/api/coverage", tags=["Coverage"])
|
||||
app.include_router(ingest.router, prefix="/api/ingest", tags=["Ingest"])
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.0
|
||||
httpx==0.27.2
|
||||
psycopg2-binary==2.9.9
|
||||
sqlalchemy==2.0.36
|
||||
pydantic==2.9.2
|
||||
pydantic-settings==2.6.1
|
||||
pyyaml==6.0.2
|
||||
python-multipart==0.0.12
|
||||
@@ -0,0 +1,273 @@
|
||||
import json
|
||||
from fastapi import APIRouter, UploadFile, File, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
from db import get_db, ParsedRule, ParserField
|
||||
from services import s1_client, rule_parser
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _star_query_texts(rule: dict) -> list[str]:
|
||||
"""
|
||||
Extract all PowerQuery/filter strings from a STAR rule.
|
||||
Handles simple rules (s1ql) and correlation rules (subQueries[].subQuery).
|
||||
"""
|
||||
texts = []
|
||||
|
||||
# Simple rules
|
||||
for field in ("s1ql", "queryLang", "query", "powerQuery"):
|
||||
v = rule.get(field)
|
||||
# queryLang "2.0" is a version string, not a query — skip short strings
|
||||
if v and isinstance(v, str) and len(v) > 5:
|
||||
texts.append(v)
|
||||
|
||||
# Correlation rules: subQueries[].subQuery
|
||||
cp = rule.get("correlationParams") or {}
|
||||
for sq in cp.get("subQueries", []):
|
||||
v = sq.get("subQuery")
|
||||
if v and isinstance(v, str):
|
||||
texts.append(v)
|
||||
# Also handle older conditions[] format
|
||||
for cond in cp.get("conditions", []):
|
||||
for key in ("filter", "query", "subQuery"):
|
||||
v = cond.get(key)
|
||||
if v and isinstance(v, str):
|
||||
texts.append(v)
|
||||
|
||||
return texts
|
||||
|
||||
|
||||
@router.post("/load-star-rules")
|
||||
async def load_star_rules(db: Session = Depends(get_db)):
|
||||
"""Fetch STAR rules from SentinelOne and index their fields."""
|
||||
try:
|
||||
rules = await s1_client.get_star_rules()
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"S1 API error: {e}")
|
||||
|
||||
# Replace all existing STAR rules cleanly to avoid duplicate key errors
|
||||
db.query(ParsedRule).filter_by(rule_type="star").delete()
|
||||
db.flush()
|
||||
|
||||
loaded = []
|
||||
for rule in rules:
|
||||
all_fields: set = set()
|
||||
for qt in _star_query_texts(rule):
|
||||
all_fields |= rule_parser.extract_star_fields(qt)
|
||||
fields = list(all_fields)
|
||||
record = ParsedRule(
|
||||
rule_id=str(rule.get("id", "")),
|
||||
name=rule.get("name", "unnamed"),
|
||||
rule_type="star",
|
||||
fields_used=fields,
|
||||
raw=json.dumps(rule),
|
||||
)
|
||||
db.add(record)
|
||||
loaded.append({"id": record.rule_id, "name": record.name, "fields": fields})
|
||||
|
||||
db.commit()
|
||||
return {"loaded": len(loaded), "rules": loaded}
|
||||
|
||||
|
||||
@router.post("/upload-sigma")
|
||||
async def upload_sigma(files: list[UploadFile] = File(...), db: Session = Depends(get_db)):
|
||||
"""Upload one or more Sigma YAML files and index their fields."""
|
||||
loaded = []
|
||||
for file in files:
|
||||
content = (await file.read()).decode("utf-8", errors="replace")
|
||||
fields = list(rule_parser.extract_sigma_fields(content))
|
||||
record = ParsedRule(
|
||||
rule_id=f"sigma_{file.filename}",
|
||||
name=file.filename or "unnamed",
|
||||
rule_type="sigma",
|
||||
fields_used=fields,
|
||||
raw=content,
|
||||
)
|
||||
db.merge(record)
|
||||
loaded.append({"name": file.filename, "fields": fields})
|
||||
|
||||
db.commit()
|
||||
return {"loaded": len(loaded), "rules": loaded}
|
||||
|
||||
|
||||
@router.post("/load-parsers-from-sdl")
|
||||
async def load_parsers_from_sdl(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Load SDL parsers from the local /app/parsers directory (mounted from ./parsers/).
|
||||
Files are placed there by the MCP-based loader or by manual copy.
|
||||
Falls back to a clear error if the directory is empty.
|
||||
"""
|
||||
import os
|
||||
parsers_dir = "/app/parsers"
|
||||
|
||||
try:
|
||||
entries = [
|
||||
e for e in os.scandir(parsers_dir)
|
||||
if e.is_file() and not e.name.startswith(".")
|
||||
]
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(503, "parsers/ directory not found — check Docker volume mount")
|
||||
|
||||
if not entries:
|
||||
raise HTTPException(
|
||||
422,
|
||||
"No parser files found in parsers/ directory. "
|
||||
"Use 'Load SDL Parsers via MCP' in Claude Code to populate it, "
|
||||
"or upload a parser file manually."
|
||||
)
|
||||
|
||||
loaded = []
|
||||
errors = []
|
||||
for entry in entries:
|
||||
try:
|
||||
with open(entry.path, "r", encoding="utf-8", errors="replace") as fh:
|
||||
content = fh.read()
|
||||
|
||||
fields: set = set()
|
||||
try:
|
||||
import json as _json
|
||||
parser_data = _json.loads(content)
|
||||
fields = rule_parser.extract_parser_fields(parser_data)
|
||||
except Exception:
|
||||
pass
|
||||
fields |= rule_parser.extract_parser_fields_from_content(content)
|
||||
|
||||
name = entry.name
|
||||
db.query(ParserField).filter_by(parser_name=name).delete()
|
||||
for f in fields:
|
||||
db.add(ParserField(parser_name=name, field_name=f, field_type="string"))
|
||||
loaded.append({"parser": name, "fields": list(fields), "field_count": len(fields)})
|
||||
except Exception as e:
|
||||
errors.append({"parser": entry.name, "error": str(e)})
|
||||
|
||||
db.commit()
|
||||
return {"loaded": len(loaded), "parsers": loaded, "errors": errors}
|
||||
|
||||
|
||||
@router.post("/upload-parser")
|
||||
async def upload_parser(file: UploadFile = File(...), db: Session = Depends(get_db)):
|
||||
"""Upload an SDL parser JSON file and index its output fields."""
|
||||
raw_bytes = await file.read()
|
||||
content_str = raw_bytes.decode("utf-8", errors="replace")
|
||||
|
||||
# Try structured JSON extraction first, fall back to content-string extraction
|
||||
fields: set = set()
|
||||
try:
|
||||
parser_data = json.loads(content_str)
|
||||
fields = rule_parser.extract_parser_fields(parser_data)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
|
||||
# Always also run content-string extraction (catches $field$ SDL format strings)
|
||||
fields |= rule_parser.extract_parser_fields_from_content(content_str)
|
||||
|
||||
db.query(ParserField).filter_by(parser_name=file.filename).delete()
|
||||
for f in fields:
|
||||
db.add(ParserField(parser_name=file.filename, field_name=f, field_type="string"))
|
||||
|
||||
db.commit()
|
||||
return {"parser": file.filename, "fields": list(fields)}
|
||||
|
||||
|
||||
class ParserContentPayload(BaseModel):
|
||||
parser_name: str
|
||||
content: str # raw SDL parser file content as string
|
||||
|
||||
|
||||
@router.post("/load-parser-content")
|
||||
async def load_parser_content(payload: ParserContentPayload, db: Session = Depends(get_db)):
|
||||
"""
|
||||
Accept raw SDL parser content (as a string) and index its output fields.
|
||||
Used by MCP-based loader scripts since the SDL HTTP API endpoint is not
|
||||
accessible from inside Docker with standard API token auth.
|
||||
"""
|
||||
fields: set = set()
|
||||
|
||||
# Try JSON parsing first (structured attributes/fields/mappings)
|
||||
try:
|
||||
parser_data = json.loads(payload.content)
|
||||
fields = rule_parser.extract_parser_fields(parser_data)
|
||||
except (json.JSONDecodeError, Exception):
|
||||
pass
|
||||
|
||||
# Always run SDL format-string extraction ($field.name$ patterns)
|
||||
fields |= rule_parser.extract_parser_fields_from_content(payload.content)
|
||||
|
||||
if not fields:
|
||||
raise HTTPException(422, "No fields could be extracted from the parser content")
|
||||
|
||||
db.query(ParserField).filter_by(parser_name=payload.parser_name).delete()
|
||||
for f in fields:
|
||||
db.add(ParserField(parser_name=payload.parser_name, field_name=f, field_type="string"))
|
||||
|
||||
db.commit()
|
||||
return {"parser": payload.parser_name, "fields": list(fields), "field_count": len(fields)}
|
||||
|
||||
|
||||
@router.get("/map")
|
||||
def get_coverage_map(db: Session = Depends(get_db)):
|
||||
"""Return coverage analysis: parser fields vs rule fields."""
|
||||
rules = db.query(ParsedRule).all()
|
||||
parser_fields_rows = db.query(ParserField).all()
|
||||
|
||||
# field → list of rules using it + data sources referenced by those rules
|
||||
rule_field_index: dict[str, list] = {}
|
||||
rule_ds_index: dict[str, set] = {} # field → set of dataSource.name values
|
||||
for rule in rules:
|
||||
query_texts = _star_query_texts(json.loads(rule.raw)) if rule.rule_type == "star" else []
|
||||
data_sources = rule_parser.extract_data_sources(query_texts)
|
||||
for field in rule.fields_used or []:
|
||||
rule_field_index.setdefault(field, []).append(
|
||||
{"rule": rule.name, "type": rule.rule_type}
|
||||
)
|
||||
rule_ds_index.setdefault(field, set()).update(data_sources)
|
||||
|
||||
# field → parser name
|
||||
parser_field_index: dict[str, str] = {
|
||||
pf.field_name: pf.parser_name for pf in parser_fields_rows
|
||||
}
|
||||
|
||||
all_fields = set(rule_field_index) | set(parser_field_index)
|
||||
|
||||
detail = {}
|
||||
for f in all_fields:
|
||||
in_parser = f in parser_field_index
|
||||
in_rules = f in rule_field_index
|
||||
detail[f] = {
|
||||
"in_parser": in_parser,
|
||||
"parser_name": parser_field_index.get(f),
|
||||
"data_sources": sorted(rule_ds_index.get(f, set())),
|
||||
"rule_count": len(rule_field_index.get(f, [])),
|
||||
"rules": rule_field_index.get(f, []),
|
||||
"status": (
|
||||
"covered" if in_parser and in_rules
|
||||
else "unused" if in_parser and not in_rules
|
||||
else "missing_parser"
|
||||
),
|
||||
}
|
||||
|
||||
parsed_unused = [f for f, d in detail.items() if d["status"] == "unused"]
|
||||
missing_parser = [f for f, d in detail.items() if d["status"] == "missing_parser"]
|
||||
covered = [f for f, d in detail.items() if d["status"] == "covered"]
|
||||
|
||||
return {
|
||||
"summary": {
|
||||
"total_parser_fields": len(parser_field_index),
|
||||
"total_rule_fields": len(rule_field_index),
|
||||
"covered": len(covered),
|
||||
"parsed_but_unused": len(parsed_unused),
|
||||
"rules_missing_parser": len(missing_parser),
|
||||
},
|
||||
"parsed_but_unused": parsed_unused,
|
||||
"rules_missing_parser": missing_parser,
|
||||
"fields": detail,
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/reset")
|
||||
def reset_data(db: Session = Depends(get_db)):
|
||||
db.query(ParsedRule).delete()
|
||||
db.query(ParserField).delete()
|
||||
db.commit()
|
||||
return {"cleared": True}
|
||||
@@ -0,0 +1,101 @@
|
||||
from datetime import datetime, timedelta
|
||||
from fastapi import APIRouter, Query, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from services import s1_client
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _date_range(days: int) -> tuple[str, str]:
|
||||
now = datetime.utcnow()
|
||||
return (
|
||||
(now - timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%S.000Z"),
|
||||
now.strftime("%Y-%m-%dT%H:%M:%S.000Z"),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/top-sources")
|
||||
async def get_top_sources(days: int = Query(7, ge=1, le=90)):
|
||||
"""Top log sources by event count over the given period."""
|
||||
from_dt, to_dt = _date_range(days)
|
||||
query = "| group events=count() by dataSource.name | sort -events | limit 25"
|
||||
try:
|
||||
result = await s1_client.run_powerquery(query, from_dt, to_dt)
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"PowerQuery error: {e}")
|
||||
return {"period_days": days, "data": result.get("events", [])}
|
||||
|
||||
|
||||
@router.get("/by-event-type")
|
||||
async def get_by_event_type(days: int = Query(7, ge=1, le=90)):
|
||||
"""Event counts grouped by source and event type."""
|
||||
from_dt, to_dt = _date_range(days)
|
||||
query = "| group events=count() by dataSource.name, event.type | sort -events | limit 100"
|
||||
try:
|
||||
result = await s1_client.run_powerquery(query, from_dt, to_dt)
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"PowerQuery error: {e}")
|
||||
return {"period_days": days, "data": result.get("events", [])}
|
||||
|
||||
|
||||
@router.get("/daily-volume")
|
||||
async def get_daily_volume(days: int = Query(7, ge=1, le=14)):
|
||||
"""Total event count per day."""
|
||||
import asyncio
|
||||
results = []
|
||||
points = min(days, 7)
|
||||
for i in range(points):
|
||||
day_from = (datetime.utcnow() - timedelta(days=i + 1)).strftime("%Y-%m-%dT00:00:00.000Z")
|
||||
day_to = (datetime.utcnow() - timedelta(days=i)).strftime("%Y-%m-%dT00:00:00.000Z")
|
||||
label = (datetime.utcnow() - timedelta(days=i + 1)).strftime("%Y-%m-%d")
|
||||
try:
|
||||
result = await s1_client.run_powerquery("| group total=count()", day_from, day_to)
|
||||
events_list = result.get("events") if isinstance(result, dict) else []
|
||||
count = events_list[0].get("total", 0) if isinstance(events_list, list) and events_list else 0
|
||||
except Exception:
|
||||
count = 0
|
||||
results.append({"date": label, "events": count})
|
||||
if i < points - 1:
|
||||
await asyncio.sleep(3)
|
||||
return list(reversed(results))
|
||||
|
||||
|
||||
class FilterRule(BaseModel):
|
||||
source: str = ""
|
||||
event_type: str = ""
|
||||
days: int = 7
|
||||
gb_per_million_events: float = 0.5
|
||||
|
||||
|
||||
@router.post("/simulate-filter")
|
||||
async def simulate_filter(rule: FilterRule):
|
||||
"""Estimate how many events and GB would be eliminated by an exclusion filter."""
|
||||
from_dt, to_dt = _date_range(rule.days)
|
||||
|
||||
clauses = []
|
||||
if rule.source:
|
||||
clauses.append(f'src.name = "{rule.source}"')
|
||||
if rule.event_type:
|
||||
clauses.append(f'event.type = "{rule.event_type}"')
|
||||
|
||||
filter_expr = " AND ".join(clauses) if clauses else "true"
|
||||
query = f"| filter {filter_expr} | count() as events"
|
||||
|
||||
try:
|
||||
result = await s1_client.run_powerquery(query, from_dt, to_dt)
|
||||
events = (result.get("events") or [{}])[0].get("events", 0) if isinstance(result.get("events"), list) else 0
|
||||
except Exception as e:
|
||||
raise HTTPException(502, f"PowerQuery error: {e}")
|
||||
|
||||
estimated_gb = round(events / 1_000_000 * rule.gb_per_million_events, 3)
|
||||
monthly_events = int(events / rule.days * 30)
|
||||
monthly_gb = round(monthly_events / 1_000_000 * rule.gb_per_million_events, 2)
|
||||
|
||||
return {
|
||||
"period_days": rule.days,
|
||||
"matched_events": events,
|
||||
"estimated_gb_period": estimated_gb,
|
||||
"projected_monthly_events": monthly_events,
|
||||
"projected_monthly_gb": monthly_gb,
|
||||
"filter": {"source": rule.source, "event_type": rule.event_type},
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
import re
|
||||
import json
|
||||
import yaml
|
||||
from typing import Set, List
|
||||
|
||||
_DS_PATTERN = re.compile(
|
||||
r"dataSource\.name\s*[=in]+\s*[\('\"]([^'\"),]+)['\")]",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
# STAR PowerQuery operators that follow a field name
|
||||
_STAR_OPS = [
|
||||
"ContainsCIS", "NotContainsCIS", "Contains", "NotContains",
|
||||
"StartsWith", "EndsWith", "In", "NotIn",
|
||||
"IsEmpty", "IsNotEmpty", "Matches", "NotMatches",
|
||||
"GreaterOrEqual", "LessOrEqual", "GreaterThan", "LessThan",
|
||||
"Between", "=", "!=",
|
||||
]
|
||||
_STAR_KEYWORD = {"and", "or", "not", "true", "false", "null"}
|
||||
_OP_PATTERN = re.compile(
|
||||
r"([\w.]+)\s*(?:" + "|".join(re.escape(op) for op in _STAR_OPS) + r")\b"
|
||||
r"|([\w.]+)\s*=", # also catch field= (no-space form used in subQuery strings)
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def extract_star_fields(query: str) -> Set[str]:
|
||||
"""Extract field names referenced in a STAR PowerQuery/subQuery string."""
|
||||
fields: Set[str] = set()
|
||||
for match in _OP_PATTERN.finditer(query):
|
||||
field = match.group(1) or match.group(2)
|
||||
if field and field.lower() not in _STAR_KEYWORD and not field[0].isdigit():
|
||||
fields.add(field)
|
||||
return fields
|
||||
|
||||
|
||||
def extract_sigma_fields(sigma_content: str) -> Set[str]:
|
||||
"""Extract field names from a Sigma rule YAML."""
|
||||
try:
|
||||
rule = yaml.safe_load(sigma_content)
|
||||
except Exception:
|
||||
return set()
|
||||
|
||||
fields: Set[str] = set()
|
||||
detection = rule.get("detection", {}) if isinstance(rule, dict) else {}
|
||||
|
||||
def _walk(node):
|
||||
if isinstance(node, dict):
|
||||
for key, val in node.items():
|
||||
if key == "condition":
|
||||
continue
|
||||
# Strip pipe modifiers: CommandLine|contains → CommandLine
|
||||
clean = key.split("|")[0]
|
||||
if clean and clean not in ("keywords",):
|
||||
fields.add(clean)
|
||||
_walk(val)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
_walk(item)
|
||||
|
||||
_walk(detection)
|
||||
return fields
|
||||
|
||||
|
||||
def extract_data_sources(texts: List[str]) -> List[str]:
|
||||
"""Extract dataSource.name values from a list of query strings."""
|
||||
sources: Set[str] = set()
|
||||
for text in texts:
|
||||
for match in _DS_PATTERN.finditer(text):
|
||||
sources.add(match.group(1).strip())
|
||||
return sorted(sources)
|
||||
|
||||
|
||||
_SDL_FIELD_PAT = re.compile(r'\$([a-zA-Z][a-zA-Z0-9._]*)(?:=[^$]*)?\$')
|
||||
_SDL_ATTR_KEY_PAT = re.compile(r'"([a-zA-Z][a-zA-Z0-9._]+)"\s*:')
|
||||
# Matches both quoted and unquoted output/to keys in rewrites:
|
||||
# output: "user.name" OR "output": "user.name"
|
||||
# "to": "src_endpoint.ip"
|
||||
_SDL_REWRITE_OUT_PAT = re.compile(
|
||||
r'(?:"output"|output|"to"|"replace")\s*:\s*"([a-zA-Z][a-zA-Z0-9._]+)"'
|
||||
)
|
||||
|
||||
|
||||
def extract_parser_fields_from_content(content: str) -> Set[str]:
|
||||
"""
|
||||
Extract output field names from SDL augmented-JSON parser content string.
|
||||
Handles:
|
||||
- $field.name$ and $field.name=pattern$ from format strings
|
||||
- "output": "field.name" and output: "field.name" from rewrites
|
||||
- quoted attribute keys from attributes{} blocks
|
||||
"""
|
||||
fields: Set[str] = set()
|
||||
|
||||
# Fields from format strings: $field.name$ or $field.name=pattern_var$
|
||||
for match in _SDL_FIELD_PAT.finditer(content):
|
||||
field = match.group(1)
|
||||
# Skip pattern variable names (no dot, short, all lowercase)
|
||||
if "." in field or field[0].isupper() or len(field) > 6:
|
||||
fields.add(field)
|
||||
|
||||
# Rewrite output targets: output: "field.name" / "output": "field.name"
|
||||
_skip_values = {"$0", "1", "2", "3", "4", "99"}
|
||||
for match in _SDL_REWRITE_OUT_PAT.finditer(content):
|
||||
val = match.group(1)
|
||||
if val not in _skip_values and "." in val:
|
||||
fields.add(val)
|
||||
|
||||
# Quoted attribute keys (skip single-word SDL builtins)
|
||||
_skip_keys = {"id", "format", "halt", "input", "output", "match", "replace",
|
||||
"timezone", "attribute", "attributes", "patterns", "formats",
|
||||
"rewrites", "type", "version"}
|
||||
for match in _SDL_ATTR_KEY_PAT.finditer(content):
|
||||
key = match.group(1)
|
||||
if key not in _skip_keys and ("." in key or len(key) > 8):
|
||||
fields.add(key)
|
||||
|
||||
return fields
|
||||
|
||||
|
||||
_SKIP_FIELD_NAMES = {
|
||||
"id", "format", "halt", "input", "output", "match", "replace",
|
||||
"timezone", "attribute", "attributes", "patterns", "formats",
|
||||
"rewrites", "type", "version", "source", "dataset", "predicate",
|
||||
"transformations", "mappings", "observables", "fields", "constant",
|
||||
"copy", "from", "to", "value", "field", "name",
|
||||
}
|
||||
|
||||
|
||||
def _extract_rewrite_fields(rewrites) -> Set[str]:
|
||||
"""Extract 'output' field names from a rewrites list."""
|
||||
fields: Set[str] = set()
|
||||
if not isinstance(rewrites, list):
|
||||
return fields
|
||||
for rw in rewrites:
|
||||
if not isinstance(rw, dict):
|
||||
continue
|
||||
# Standard SDL rewrite: {"input": "...", "output": "field.name"}
|
||||
out = rw.get("output") or rw.get("to")
|
||||
if out and isinstance(out, str) and "." in out and out not in _SKIP_FIELD_NAMES:
|
||||
fields.add(out)
|
||||
return fields
|
||||
|
||||
|
||||
def _walk_mappings(node) -> Set[str]:
|
||||
"""Recursively extract copy.to and constant.field from SDL mappings blocks."""
|
||||
fields: Set[str] = set()
|
||||
if isinstance(node, dict):
|
||||
# transformations copy: {"copy": {"from": "...", "to": "field.name"}}
|
||||
if "copy" in node and isinstance(node["copy"], dict):
|
||||
to = node["copy"].get("to")
|
||||
if to and isinstance(to, str) and "." in to:
|
||||
fields.add(to)
|
||||
# transformations constant: {"constant": {"value": ..., "field": "field.name"}}
|
||||
if "constant" in node and isinstance(node["constant"], dict):
|
||||
f = node["constant"].get("field")
|
||||
if f and isinstance(f, str) and "." in f:
|
||||
fields.add(f)
|
||||
for v in node.values():
|
||||
fields |= _walk_mappings(v)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
fields |= _walk_mappings(item)
|
||||
return fields
|
||||
|
||||
|
||||
def extract_parser_fields(parser_json: dict) -> Set[str]:
|
||||
"""
|
||||
Extract output field names from an SDL parser JSON dict.
|
||||
Handles: attributes lists, fields lists, mappings targets,
|
||||
rewrites[].output, rewrites[].to, copy.to, constant.field.
|
||||
"""
|
||||
fields: Set[str] = set()
|
||||
|
||||
# Legacy: attributes as list of {name: ...}
|
||||
for attr in parser_json.get("attributes", []):
|
||||
if isinstance(attr, dict) and "name" in attr:
|
||||
fields.add(attr["name"])
|
||||
|
||||
# Legacy: fields list
|
||||
for field in parser_json.get("fields", []):
|
||||
if isinstance(field, str):
|
||||
fields.add(field)
|
||||
elif isinstance(field, dict) and "name" in field:
|
||||
fields.add(field["name"])
|
||||
|
||||
# Legacy: flat mappings list with "target"
|
||||
for mapping in parser_json.get("mappings", []):
|
||||
if isinstance(mapping, dict) and "target" in mapping:
|
||||
fields.add(mapping["target"])
|
||||
|
||||
# SDL rewrites[].output in top-level formats[]
|
||||
for fmt in parser_json.get("formats", []):
|
||||
if isinstance(fmt, dict):
|
||||
fields |= _extract_rewrite_fields(fmt.get("rewrites", []))
|
||||
|
||||
# SDL mappings block (nested transformations with copy.to / constant.field)
|
||||
mappings_block = parser_json.get("mappings", {})
|
||||
if isinstance(mappings_block, dict):
|
||||
fields |= _walk_mappings(mappings_block)
|
||||
|
||||
# observables[].name
|
||||
for obs in parser_json.get("observables", {}).get("fields", []):
|
||||
if isinstance(obs, dict) and "name" in obs:
|
||||
n = obs["name"]
|
||||
if "." in n:
|
||||
fields.add(n)
|
||||
|
||||
return fields
|
||||
@@ -0,0 +1,135 @@
|
||||
import os
|
||||
import asyncio
|
||||
import httpx
|
||||
from datetime import datetime, timezone
|
||||
|
||||
BASE_URL = os.environ.get("S1_BASE_URL", "https://demo.sentinelone.net").rstrip("/")
|
||||
TOKEN = os.environ.get("S1_API_TOKEN", "")
|
||||
|
||||
# Scalyr/XDR PowerQuery credentials — from SDL_XDR_URL + SDL_LOG_READ_KEY
|
||||
# in the SentinelOne console: Settings → Integrations → Data Lake API Keys
|
||||
SDL_XDR_URL = os.environ.get("SDL_XDR_URL", "https://xdr.us1.sentinelone.net").rstrip("/")
|
||||
SDL_LOG_READ_KEY = os.environ.get("SDL_LOG_READ_KEY", "")
|
||||
|
||||
# Management Console API uses ApiToken auth
|
||||
HEADERS = {
|
||||
"Authorization": f"ApiToken {TOKEN}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def _iso_to_epoch_ms(iso_str: str) -> int:
|
||||
"""Convert ISO-8601 UTC string to epoch milliseconds for Scalyr API."""
|
||||
dt = datetime.fromisoformat(iso_str.replace("Z", "+00:00"))
|
||||
return int(dt.timestamp() * 1000)
|
||||
|
||||
|
||||
async def get_star_rules(limit: int = 200) -> list:
|
||||
"""Fetch active STAR rules from the Management Console API."""
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.get(
|
||||
f"{BASE_URL}/web/api/v2.1/cloud-detection/rules",
|
||||
headers=HEADERS,
|
||||
params={"limit": limit},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("data", [])
|
||||
|
||||
|
||||
async def run_powerquery(query: str, from_date: str, to_date: str) -> dict:
|
||||
"""
|
||||
Run a PowerQuery against the Singularity Data Lake via the Scalyr XDR API.
|
||||
Uses SDL_XDR_URL + SDL_LOG_READ_KEY (Scalyr readlog token).
|
||||
The Scalyr PowerQuery API is synchronous — results return in one request.
|
||||
"""
|
||||
if not SDL_LOG_READ_KEY:
|
||||
return {"events": [], "error": "SDL_LOG_READ_KEY not configured — add it to .env"}
|
||||
|
||||
start_ms = _iso_to_epoch_ms(from_date)
|
||||
end_ms = _iso_to_epoch_ms(to_date)
|
||||
|
||||
payload = {
|
||||
"token": SDL_LOG_READ_KEY,
|
||||
"query": query,
|
||||
"startTime": start_ms,
|
||||
"endTime": end_ms,
|
||||
"maxCount": 1000,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
for attempt in range(3):
|
||||
try:
|
||||
resp = await client.post(
|
||||
f"{SDL_XDR_URL}/api/powerQuery",
|
||||
json=payload,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
break
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 429 and attempt < 2:
|
||||
await asyncio.sleep(10 * (attempt + 1))
|
||||
continue
|
||||
raise RuntimeError(
|
||||
f"HTTP {e.response.status_code} from {e.request.url}: {e.response.text[:500]}"
|
||||
) from e
|
||||
|
||||
data = resp.json()
|
||||
status = data.get("status", "")
|
||||
|
||||
if status != "success":
|
||||
# Return full response as error detail for debugging
|
||||
return {"events": [], "error": f"PowerQuery status={status}: {str(data)[:400]}"}
|
||||
|
||||
# Scalyr PowerQuery returns: {"status":"success","columns":[{"name":"..."},...], "values":[[...],...],...}
|
||||
raw_cols = data.get("columns", [])
|
||||
values = data.get("values", [])
|
||||
|
||||
if raw_cols and values:
|
||||
# columns may be list of strings or list of {"name":...} dicts
|
||||
col_names = [
|
||||
c["name"] if isinstance(c, dict) else c
|
||||
for c in raw_cols
|
||||
]
|
||||
rows = [dict(zip(col_names, row)) for row in values]
|
||||
return {"events": rows}
|
||||
|
||||
# Fallback: return raw matches array
|
||||
matches = data.get("matches", [])
|
||||
return {"events": matches}
|
||||
|
||||
|
||||
async def list_sdl_parsers() -> list[str]:
|
||||
"""List all parser filenames under /logParsers/ in SDL."""
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.get(
|
||||
f"{BASE_URL}/api/v1/files/logParsers",
|
||||
headers=HEADERS,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
# Response is a list of file objects or a dict with 'files' key
|
||||
if isinstance(data, list):
|
||||
return [f.get("name") or f.get("path", "") for f in data if isinstance(f, dict)]
|
||||
return [f.get("name") or f.get("path", "") for f in data.get("files", [])]
|
||||
|
||||
|
||||
async def get_sdl_parser(filename: str) -> dict:
|
||||
"""Fetch a single SDL parser file by name."""
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.get(
|
||||
f"{BASE_URL}/api/v1/files/logParsers/{filename}",
|
||||
headers=HEADERS,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def get_sites() -> list:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.get(
|
||||
f"{BASE_URL}/web/api/v2.1/sites",
|
||||
headers=HEADERS,
|
||||
params={"limit": 100},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json().get("data", {}).get("sites", [])
|
||||
Reference in New Issue
Block a user