From d8d62478c05e06a1b3c31d64976cc64c0e488a2a Mon Sep 17 00:00:00 2001 From: marc Date: Wed, 20 May 2026 19:41:00 +0200 Subject: [PATCH] Add helper scripts: SDL parser sync, PQ probes, test-parser smoke tests --- tools/probe_avelios.py | 100 ++++++++++++++++++++++++++++++++++ tools/probe_avelios_fields.py | 89 ++++++++++++++++++++++++++++++ tools/probe_avelios_wide.py | 53 ++++++++++++++++++ tools/probe_pq_syntax.py | 77 ++++++++++++++++++++++++++ tools/sdl_config.example.json | 7 +++ tools/sync_sdl_parsers.py | 100 ++++++++++++++++++++++++++++++++++ tools/test_avelios_multi.py | 27 +++++++++ tools/test_avelios_parser.py | 22 ++++++++ 8 files changed, 475 insertions(+) create mode 100644 tools/probe_avelios.py create mode 100644 tools/probe_avelios_fields.py create mode 100644 tools/probe_avelios_wide.py create mode 100644 tools/probe_pq_syntax.py create mode 100644 tools/sdl_config.example.json create mode 100644 tools/sync_sdl_parsers.py create mode 100644 tools/test_avelios_multi.py create mode 100644 tools/test_avelios_parser.py diff --git a/tools/probe_avelios.py b/tools/probe_avelios.py new file mode 100644 index 0000000..a1b3e5c --- /dev/null +++ b/tools/probe_avelios.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Probe the SDL tenant to understand why Avelios Medical field-population shows 0%.""" +import json, time, urllib.request, urllib.error +import os + +def _load_sdl_cfg(): + import json as _j, os as _o, sys as _s + here = _o.path.dirname(_o.path.abspath(__file__)) + candidates = [ + _o.environ.get("SDL_CONFIG"), + _o.path.join(here, "sdl_config.json"), + _o.path.join(here, "..", "sdl_config.json"), + ] + for p in candidates: + if p and _o.path.exists(p): + with open(p) as fh: + return _j.load(fh) + _s.stderr.write( + "ERROR: no SDL config found. Set $SDL_CONFIG or create sdl_config.json " + "(see sdl_config.example.json)\n") + _s.exit(2) + + +CFG = _load_sdl_cfg() +BASE = CFG['base_url'].rstrip('/') +KEY = CFG['log_read_key'] +END_MS = int(time.time() * 1000) +START_MS = END_MS - 24 * 3600 * 1000 # last 24h + + +def pq(query: str, max_count: int = 10) -> dict: + body = json.dumps({ + "token": KEY, "query": query, + "startTime": START_MS, "endTime": END_MS, + "maxCount": max_count, + }).encode() + req = urllib.request.Request(BASE + '/api/powerQuery', data=body, + headers={"Content-Type": "application/json"}) + try: + return json.loads(urllib.request.urlopen(req, timeout=30).read()) + except urllib.error.HTTPError as e: + return {"_err": f"HTTP {e.code}: {e.read().decode()[:200]}"} + except Exception as e: + return {"_err": str(e)[:200]} + + +def show(label, d): + if "_err" in d: + print(f"[ERR] {label}: {d['_err']}"); return + cols = [c['name'] if isinstance(c, dict) else c for c in d.get('columns', [])] + vals = d.get('values', []) or d.get('matches', []) + print(f"[OK ] {label} cols={cols} rows={len(vals)}") + for v in vals[:8]: + print(f" {v}") + + +# 1. Distinct dataSource.name values containing 'velio' +print("=" * 70) +print("1. Source-name spellings containing 'velio'") +print("=" * 70) +show("by dataSource.name", + pq("| group n=count() by dataSource.name | sort -n | limit 50", max_count=50)) + +# 2. Try a few candidate names +print() +print("=" * 70) +print("2. Try filtering by candidate names") +print("=" * 70) +for cand in ["Avelios Medical", "Avelios-Medical", "Avelios-Medical-OCSF", + "avelios", "Avelios"]: + d = pq(f"| filter dataSource.name == '{cand}' | group n=count()", max_count=1) + n = (d.get('values') or [[None]])[0][0] if 'values' in d else d + print(f" {cand!r:<35} -> {n}") +for cand in ["Avelios Medical", "Avelios-Medical-OCSF", "avelios"]: + d = pq(f"| filter dataSource.name contains '{cand}' | group n=count()", max_count=1) + n = (d.get('values') or [[None]])[0][0] if 'values' in d else d + print(f" contains {cand!r:<25} -> {n}") + +# 3. Sample one raw event to see what column names actually come back +print() +print("=" * 70) +print("3. Sample one event — what keys/columns are returned?") +print("=" * 70) +d = pq("| filter dataSource.name contains 'velio' | limit 1", max_count=1) +if "_err" in d: + print(" ", d["_err"]) +else: + print(" columns:", [c['name'] if isinstance(c, dict) else c for c in d.get('columns', [])][:30]) + print(" first row sample:", str((d.get('values') or [None])[0])[:400]) + +# 4. If we got columns, check which OCSF fields exist +print() +print("=" * 70) +print("4. Field presence in last 24h for Avelios (using columns command)") +print("=" * 70) +d = pq("| filter dataSource.name contains 'velio' | " + "columns dataSource.name, metadata.product.name, metadata.event_code, " + "actor.user.name, src_endpoint.ip, dst_endpoint.ip | limit 5", + max_count=5) +show("columns view", d) diff --git a/tools/probe_avelios_fields.py b/tools/probe_avelios_fields.py new file mode 100644 index 0000000..8e8d22d --- /dev/null +++ b/tools/probe_avelios_fields.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Inspect Avelios Medical events: one query, full row dump, then field stats from Python.""" +import json, time, urllib.request, collections +import os + +def _load_sdl_cfg(): + import json as _j, os as _o, sys as _s + here = _o.path.dirname(_o.path.abspath(__file__)) + candidates = [ + _o.environ.get("SDL_CONFIG"), + _o.path.join(here, "sdl_config.json"), + _o.path.join(here, "..", "sdl_config.json"), + ] + for p in candidates: + if p and _o.path.exists(p): + with open(p) as fh: + return _j.load(fh) + _s.stderr.write( + "ERROR: no SDL config found. Set $SDL_CONFIG or create sdl_config.json " + "(see sdl_config.example.json)\n") + _s.exit(2) + + +CFG = _load_sdl_cfg() +BASE, KEY = CFG['base_url'].rstrip('/'), CFG['log_read_key'] +NOW = int(time.time() * 1000) +START = NOW - 72 * 3600 * 1000 # last 3 days + + +def pq(query, mc=200): + body = json.dumps({"token": KEY, "query": query, + "startTime": START, "endTime": NOW, + "maxCount": mc}).encode() + req = urllib.request.Request(BASE + '/api/powerQuery', data=body, + headers={"Content-Type": "application/json"}) + return json.loads(urllib.request.urlopen(req, timeout=60).read()) + + +print("Fetching Avelios Medical sample (max 200, last 72h) ...") +d = pq("| filter dataSource.name == 'Avelios Medical' | limit 200") +cols = [c['name'] if isinstance(c, dict) else c for c in d.get('columns', [])] +vals = d.get('values', []) or [] +print(f"Columns returned ({len(cols)}): {cols}") +print(f"Rows: {len(vals)}") +print() + +# Tally non-null rate per returned column +counts = {c: 0 for c in cols} +for row in vals: + for c, v in zip(cols, row): + if v not in (None, '', 'null'): + counts[c] += 1 +print("=== Column populated-rate (out of returned columns) ===") +for c in cols: + n = counts[c] + pct = round(100 * n / max(1, len(vals)), 1) + print(f" {c:<35} {n:>4} / {len(vals)} {pct:>5}%") + +print() +print("=== First 2 events (pretty) ===") +for row in vals[:2]: + print(json.dumps(dict(zip(cols, row)), indent=2, default=str)[:1500]) + print("---") + +print() +print("=== Distinct fields IN the message body (if JSON) ===") +# If the events carry a structured body, peek inside it +field_freq = collections.Counter() +for row in vals: + rd = dict(zip(cols, row)) + msg = rd.get('message') or rd.get('body') or rd.get('attributes') + if isinstance(msg, str): + try: + j = json.loads(msg) + except Exception: + continue + else: + j = msg + if isinstance(j, dict): + def walk(obj, prefix=''): + for k, v in obj.items(): + key = f"{prefix}.{k}" if prefix else k + if isinstance(v, dict): + walk(v, key) + else: + field_freq[key] += 1 + walk(j) +for k, c in field_freq.most_common(40): + print(f" {k:<45} in {c:>3} events") diff --git a/tools/probe_avelios_wide.py b/tools/probe_avelios_wide.py new file mode 100644 index 0000000..86bf857 --- /dev/null +++ b/tools/probe_avelios_wide.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Search wider windows for Avelios Medical events.""" +import json, time, urllib.request +import os + +def _load_sdl_cfg(): + import json as _j, os as _o, sys as _s + here = _o.path.dirname(_o.path.abspath(__file__)) + candidates = [ + _o.environ.get("SDL_CONFIG"), + _o.path.join(here, "sdl_config.json"), + _o.path.join(here, "..", "sdl_config.json"), + ] + for p in candidates: + if p and _o.path.exists(p): + with open(p) as fh: + return _j.load(fh) + _s.stderr.write( + "ERROR: no SDL config found. Set $SDL_CONFIG or create sdl_config.json " + "(see sdl_config.example.json)\n") + _s.exit(2) + + +CFG = _load_sdl_cfg() +BASE, KEY = CFG['base_url'].rstrip('/'), CFG['log_read_key'] +NOW = int(time.time() * 1000) + + +def pq(query, start_ms, end_ms, mc=5): + body = json.dumps({"token": KEY, "query": query, + "startTime": start_ms, "endTime": end_ms, + "maxCount": mc}).encode() + req = urllib.request.Request(BASE + '/api/powerQuery', data=body, + headers={"Content-Type": "application/json"}) + try: + return json.loads(urllib.request.urlopen(req, timeout=60).read()) + except Exception as e: + return {"_err": str(e)[:200]} + + +for days in (1, 3, 7): + start = NOW - days * 24 * 3600 * 1000 + print(f"\n=== last {days}d ===") + d = pq("| group n=count() by dataSource.name | sort -n | limit 30", start, NOW, mc=30) + if "_err" in d: + print(d["_err"]); continue + for row in d.get("values", []): + name = row[0] + if name and "velio" in name.lower(): + print(f" HIT: {row}") + # show top 10 in this window + for row in (d.get("values", []) or [])[:10]: + print(f" {row}") diff --git a/tools/probe_pq_syntax.py b/tools/probe_pq_syntax.py new file mode 100644 index 0000000..128b40d --- /dev/null +++ b/tools/probe_pq_syntax.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Probe what PowerQuery syntax this SDL tenant accepts.""" +import json, time, urllib.request, urllib.error, sys +import os + +def _load_sdl_cfg(): + import json as _j, os as _o, sys as _s + here = _o.path.dirname(_o.path.abspath(__file__)) + candidates = [ + _o.environ.get("SDL_CONFIG"), + _o.path.join(here, "sdl_config.json"), + _o.path.join(here, "..", "sdl_config.json"), + ] + for p in candidates: + if p and _o.path.exists(p): + with open(p) as fh: + return _j.load(fh) + _s.stderr.write( + "ERROR: no SDL config found. Set $SDL_CONFIG or create sdl_config.json " + "(see sdl_config.example.json)\n") + _s.exit(2) + + +CFG = _load_sdl_cfg() +URL = CFG['base_url'].rstrip('/') + '/api/powerQuery' +END_MS = int(time.time() * 1000) +START_MS = END_MS - 3600 * 1000 # last hour + + +def run(label: str, query: str): + body = json.dumps({ + "token": CFG['log_read_key'], + "query": query, + "startTime": START_MS, + "endTime": END_MS, + "maxCount": 5, + }).encode() + req = urllib.request.Request(URL, data=body, headers={"Content-Type": "application/json"}) + try: + resp = urllib.request.urlopen(req, timeout=30).read() + d = json.loads(resp) + st = d.get('status', '?') + cols = d.get('columns') or [] + vals = d.get('values') or d.get('matches') or [] + print(f"[OK ] {label:<40} status={st} cols={len(cols)} rows={len(vals)}") + if vals: + print(f" sample={str(vals[0])[:160]}") + except urllib.error.HTTPError as e: + body = e.read().decode() + try: + j = json.loads(body) + msg = j.get('message', body)[:200] + except Exception: + msg = body[:200] + print(f"[ERR] {label:<40} HTTP {e.code}: {msg}") + except Exception as e: + print(f"[ERR] {label:<40} {type(e).__name__}: {str(e)[:160]}") + + +CASES = [ + ("leading-pipe single-stage", "| group total=count()"), + ("no-pipe single-stage", "group total=count()"), + ("leading-pipe multi-stage", "| group events=count() by dataSource.name | sort -events | limit 5"), + ("no-pipe multi-stage", "group events=count() by dataSource.name | sort -events | limit 5"), + ("no-pipe trim sort", "group events=count() by dataSource.name | limit 5"), + ("filter then group", "dataSource.name=='SentinelOne' | group events=count()"), + ("filter (modern keyword)", "filter dataSource.name=='SentinelOne' | group events=count()"), + ("dataset-style with sort", "group events=count() by dataSource.name | sort events desc | limit 5"), + ("count() as alias", "| count() as events"), + ("group by event.type", "group events=count() by event.type | limit 5"), +] + +print(f"URL: {URL}") +print(f"Window: last 1h ({START_MS}..{END_MS} ms)") +print() +for label, q in CASES: + run(label, q) diff --git a/tools/sdl_config.example.json b/tools/sdl_config.example.json new file mode 100644 index 0000000..0c96307 --- /dev/null +++ b/tools/sdl_config.example.json @@ -0,0 +1,7 @@ +{ + "_comment": "Copy to sdl_config.json (or set $SDL_CONFIG to its path). Only the keys you need are required. log_read_key for queries; config_read_key for listFiles/getFile (parser sync).", + "base_url": "https://xdr.us1.sentinelone.net", + "log_read_key": "REPLACE_WITH_LOG_READ_KEY", + "config_read_key": "REPLACE_WITH_CONFIG_READ_KEY", + "console_api_token": "REPLACE_WITH_CONSOLE_API_TOKEN_OR_LEAVE_BLANK" +} diff --git a/tools/sync_sdl_parsers.py b/tools/sync_sdl_parsers.py new file mode 100644 index 0000000..675745a --- /dev/null +++ b/tools/sync_sdl_parsers.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +""" +Pull every parser under /logParsers/ from the SDL tenant and drop it into +./parsers/ so the SIEM-Toolkit Parser Test Runner can list it. + +Auth: config_read_key from sentinelone-sdl-api/config.json +""" +from __future__ import annotations +import json +import os +import sys +import urllib.request +import urllib.parse +import urllib.error + +def _load_sdl_cfg(): + import json as _j, os as _o, sys as _s + here = _o.path.dirname(_o.path.abspath(__file__)) + candidates = [ + _o.environ.get("SDL_CONFIG"), + _o.path.join(here, "sdl_config.json"), + _o.path.join(here, "..", "sdl_config.json"), + ] + for p in candidates: + if p and _o.path.exists(p): + with open(p) as fh: + return _j.load(fh) + _s.stderr.write( + "ERROR: no SDL config found. Set $SDL_CONFIG or create sdl_config.json " + "(see sdl_config.example.json)\n") + _s.exit(2) + + +SDL_CFG_PATH = os.environ.get('SDL_CONFIG') # placeholder; cfg loaded below +DEST = os.environ.get('PARSERS_DIR', os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'parsers')) +def call(base_url: str, token: str, path: str, params: dict) -> dict: + """POST with JSON body — works for both listFiles and getFile on SDL.""" + url = f"{base_url.rstrip('/')}{path}" + body = json.dumps({**params, "token": token}).encode() + req = urllib.request.Request(url, data=body, headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + }) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return json.loads(r.read()) + except urllib.error.HTTPError as e: + body = e.read().decode(errors="replace")[:300] + raise RuntimeError(f"HTTP {e.code} {path}: {body}") + + +def main() -> int: + cfg = _load_sdl_cfg() + base = cfg["base_url"] + # config_read_key first (per docs), fall back to console_api_token + token = cfg.get("config_read_key") or cfg.get("console_api_token") + if not token: + print("No config_read_key or console_api_token in config.json", file=sys.stderr) + return 2 + + print(f"Listing /logParsers/ from {base} ...") + res = call(base, token, "/api/listFiles", {"pathPrefix": "/logParsers/"}) + paths = res.get("paths", []) + print(f"Found {len(paths)} files under /logParsers/") + + os.makedirs(DEST, exist_ok=True) + fetched, skipped, failed = 0, 0, [] + + for p in paths: + # Strip leading /logParsers/, sanitize for filesystem + name = p.rsplit("/", 1)[-1] or "_unnamed" + # Avoid colliding with existing sample files? Always overwrite to keep fresh. + try: + r = call(base, token, "/api/getFile", {"path": p}) + except Exception as e: + failed.append((p, str(e))) + continue + + content = r.get("content") + if content is None: + failed.append((p, "no content")) + continue + + out = os.path.join(DEST, name) + with open(out, "w", encoding="utf-8") as fh: + fh.write(content) + ver = r.get("version", "?") + print(f" + {name:<60} v{ver} ({len(content)} bytes)") + fetched += 1 + + print() + print(f"Done: fetched={fetched}, failed={len(failed)}") + if failed: + for p, err in failed[:10]: + print(f" ! {p}: {err}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/test_avelios_multi.py b/tools/test_avelios_multi.py new file mode 100644 index 0000000..c3f2ffd --- /dev/null +++ b/tools/test_avelios_multi.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Verify the Parser Test Runner accepts multi-line NDJSON for JSON-mode parsers.""" +import json, urllib.request +import os + +LINES = [ + '{"timestamp":"2026-05-14T00:00:41.969Z","event_type":"DATA_IMPORT_COMPLETED","event_category":"data_transfer","severity":"INFO","outcome":"success","source":{"application":"Avelios Medical"}}', + '{"timestamp":"2026-05-14T00:07:41.969Z","event_type":"PERFORMANCE_DEGRADATION","event_category":"system","severity":"MEDIUM","outcome":"success","source":{"application":"Avelios Medical"}}', + '{"timestamp":"2026-05-14T00:24:41.969Z","event_type":"MALWARE_DETECTED","event_category":"security","severity":"CRITICAL","outcome":"detected","source":{"application":"Avelios Medical"},"details":{"malware_name":"Trojan.GenericKD"}}', +] + +body = json.dumps({"parser_name": "Avelios-Medical-OCSF", "log_line": "\n".join(LINES)}).encode() +req = urllib.request.Request( + "http://localhost:8001/api/quality/test-parser", + data=body, headers={"Content-Type": "application/json"}) +r = json.loads(urllib.request.urlopen(req, timeout=30).read()) + +print(f"matched = {r.get('matched')}") +print(f"mode = {r.get('mode')}") +print(f"payloads = {r.get('payload_count')} (showing {r.get('showing_payload')})") +print(f"extracted = {r.get('extracted_count')}") +print(f"derived = {r.get('derived_count')}") +print(f"parse_errors = {r.get('parse_errors')}") +print() +print("rewrites applied (first payload):") +for rw in r.get("rewrites_applied", [])[:10]: + print(f" {rw['input']:<18} -> {rw['output']:<28} = {rw['result']!r}") diff --git a/tools/test_avelios_parser.py b/tools/test_avelios_parser.py new file mode 100644 index 0000000..48afcc2 --- /dev/null +++ b/tools/test_avelios_parser.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 +import json, urllib.request +import os + +log = '{"timestamp": "2026-05-14T00:24:41.969Z", "event_id": "d5c76dd2-5320-4b32-bd27-09acedfb5fdb", "event_type": "MALWARE_DETECTED", "event_category": "security", "severity": "CRITICAL", "source": {"application": "Avelios Medical", "module": "SecurityMonitor"}, "outcome": "detected", "details": {"malware_name": "Trojan.GenericKD"}}' + +body = json.dumps({"parser_name": "Avelios-Medical-OCSF", "log_line": log}).encode() +req = urllib.request.Request( + "http://localhost:8001/api/quality/test-parser", + data=body, headers={"Content-Type": "application/json"}) +r = json.loads(urllib.request.urlopen(req, timeout=30).read()) + +print(f"matched={r.get('matched')} mode={r.get('mode')} " + f"extracted={r.get('extracted_count')} derived={r.get('derived_count')}") +print() +print("json-extract fields (first 12):") +for f in [x for x in r.get("fields", []) if x.get("source") == "json-extract"][:12]: + print(f" {f['field']:<32} = {str(f['value'])[:50]}") +print() +print("rewrites applied:") +for rw in r.get("rewrites_applied", [])[:12]: + print(f" {rw['input']:<18} -> {rw['output']:<28} = {rw['result']!r}")