mirror of
https://github.com/marcredhat/SIEM-toolkit-patched
synced 2026-06-08 12:33:51 +00:00
Add unlabelled event detection, stub parser quality, Sync All, and modern UI redesign
Key changes: - Unlabelled event banner: shows count only after Sample Events is clicked; uses broad SDL filter expression; time window synced to sync-days dropdown - Parser Quality: new "Attributes Missing" subsection listing all parsers without dataSource.name regardless of event volume - Coverage map: filter buttons (All / Complete Parser / Attributes Missing); stat card renamed to "Incomplete Parser"; stub count excluded from sync when no active sources - Sync All button: runs SDL parser sync → library sync → live sources sync in sequence - Reset now clears ActiveSource table and resets unlabelled count cache - run_powerquery: configurable max_count param (default 1000, 50M for count queries) - _DS_NAME_RE: supports both quoted and unquoted dataSource.name keys in parser files - Full modern UI redesign: slate palette, gradient cards, ring borders, pill nav, colored stat accents - Updated 7 tracked parser files synced from SDL Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Mick
+2
-1
@@ -1,5 +1,5 @@
|
||||
import os
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, Text
|
||||
from sqlalchemy import create_engine, Column, Integer, String, Float, DateTime, Text, Boolean
|
||||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import declarative_base, sessionmaker
|
||||
from datetime import datetime
|
||||
@@ -37,6 +37,7 @@ class ActiveSource(Base):
|
||||
event_count = Column(Integer, default=0)
|
||||
synced_at = Column(DateTime, default=datetime.utcnow)
|
||||
parser_detected = Column(Integer, default=0) # >0 means parsed events seen in data lake
|
||||
unlabelled = Column(Boolean, default=False) # True = events had no dataSource.name
|
||||
|
||||
|
||||
class IngestSnapshot(Base):
|
||||
|
||||
@@ -11,6 +11,9 @@ with engine.connect() as _conn:
|
||||
_conn.execute(text(
|
||||
"ALTER TABLE active_sources ADD COLUMN IF NOT EXISTS parser_detected INTEGER DEFAULT 0"
|
||||
))
|
||||
_conn.execute(text(
|
||||
"ALTER TABLE active_sources ADD COLUMN IF NOT EXISTS unlabelled BOOLEAN DEFAULT FALSE"
|
||||
))
|
||||
_conn.commit()
|
||||
|
||||
app = FastAPI(title="SIEM Toolkit", version="1.0.0")
|
||||
|
||||
+173
-16
@@ -207,16 +207,109 @@ async def upload_sigma(files: list[UploadFile] = File(...), db: Session = Depend
|
||||
return {"loaded": len(loaded), "rules": loaded}
|
||||
|
||||
|
||||
def _fetch_parsers_from_console(parsers_dir: str) -> dict:
|
||||
"""
|
||||
Fetch every parser under /logParsers/ from the SDL console and write them
|
||||
to parsers_dir. Uses SDL_CONFIG_READ_KEY (needs 'Manage config files' permission)
|
||||
and SDL_XDR_URL from the environment.
|
||||
|
||||
Returns {"fetched": N, "failed": [...], "skipped": reason_or_None}
|
||||
"""
|
||||
import urllib.request, urllib.error, json as _json, os as _os
|
||||
|
||||
# Read live from .env file so Settings-page saves are picked up without restart
|
||||
def _env_val(key: str) -> str:
|
||||
val = _os.environ.get(key, "")
|
||||
if not val:
|
||||
env_path = _os.environ.get("ENV_FILE_PATH", "/app/.env")
|
||||
try:
|
||||
for line in open(env_path).read().splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, _, v = line.partition("=")
|
||||
if k.strip() == key:
|
||||
val = v.strip()
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
return val
|
||||
|
||||
config_key = _env_val("SDL_CONFIG_READ_KEY")
|
||||
base_url = _env_val("SDL_XDR_URL").rstrip("/")
|
||||
|
||||
if not config_key:
|
||||
return {"fetched": 0, "failed": [], "skipped": "SDL_CONFIG_READ_KEY not set"}
|
||||
if not base_url:
|
||||
return {"fetched": 0, "failed": [], "skipped": "SDL_XDR_URL not set"}
|
||||
|
||||
def _post(path: str, params: dict) -> dict:
|
||||
url = f"{base_url}{path}"
|
||||
body = _json.dumps({**params, "token": config_key}).encode()
|
||||
req = urllib.request.Request(url, data=body, headers={
|
||||
"Authorization": f"Bearer {config_key}",
|
||||
"Content-Type": "application/json",
|
||||
})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return _json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
err_body = e.read().decode(errors="replace")[:300]
|
||||
raise RuntimeError(f"HTTP {e.code} {path}: {err_body}")
|
||||
|
||||
# List all parser paths
|
||||
res = _post("/api/listFiles", {"pathPrefix": "/logParsers/"})
|
||||
|
||||
# Support multiple response shapes: {"paths": [...]} or {"files": [...]}
|
||||
raw_paths = res.get("paths") or res.get("files") or []
|
||||
|
||||
# Each element may be a plain string or a dict with a "path"/"name" key
|
||||
paths = []
|
||||
for p in raw_paths:
|
||||
if isinstance(p, dict):
|
||||
p = p.get("path") or p.get("name") or ""
|
||||
if isinstance(p, str) and p.startswith("/logParsers/"):
|
||||
paths.append(p)
|
||||
|
||||
_os.makedirs(parsers_dir, exist_ok=True)
|
||||
fetched, failed = 0, []
|
||||
|
||||
for p in paths:
|
||||
name = p.rsplit("/", 1)[-1] or "_unnamed"
|
||||
try:
|
||||
r = _post("/api/getFile", {"path": p})
|
||||
content = r.get("content")
|
||||
if content is None:
|
||||
failed.append({"path": p, "error": "no content", "raw": r})
|
||||
continue
|
||||
with open(_os.path.join(parsers_dir, name), "w", encoding="utf-8") as fh:
|
||||
fh.write(content)
|
||||
fetched += 1
|
||||
except Exception as e:
|
||||
failed.append({"path": p, "error": str(e)})
|
||||
|
||||
# Surface the raw API response so callers can see exactly what was returned.
|
||||
# Truncate paths list so the response stays readable (first 200).
|
||||
debug_info = {
|
||||
"response_keys": list(res.keys()),
|
||||
"paths_found": len(paths),
|
||||
"paths_listed": paths[:200],
|
||||
}
|
||||
return {"fetched": fetched, "failed": failed, "skipped": None, "debug": debug_info}
|
||||
|
||||
|
||||
@router.post("/load-parsers-from-sdl")
|
||||
async def load_parsers_from_sdl(db: Session = Depends(get_db)):
|
||||
"""
|
||||
Load SDL parsers from the local /app/parsers directory (mounted from ./parsers/).
|
||||
Files are placed there by the MCP-based loader or by manual copy.
|
||||
Falls back to a clear error if the directory is empty.
|
||||
Sync SDL parsers from the console (if SDL_CONFIG_READ_KEY is set) then index
|
||||
every file in the local /app/parsers directory into the DB.
|
||||
"""
|
||||
import os
|
||||
parsers_dir = "/app/parsers"
|
||||
|
||||
# ── Step 1: fetch from console (best-effort) ────────────────────────────
|
||||
fetch_result = _fetch_parsers_from_console(parsers_dir)
|
||||
|
||||
# ── Step 2: load whatever is on disk into the DB ─────────────────────────
|
||||
try:
|
||||
entries = [
|
||||
e for e in os.scandir(parsers_dir)
|
||||
@@ -225,12 +318,19 @@ async def load_parsers_from_sdl(db: Session = Depends(get_db)):
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(503, "parsers/ directory not found — check Docker volume mount")
|
||||
|
||||
if not entries and fetch_result["skipped"]:
|
||||
raise HTTPException(
|
||||
422,
|
||||
f"No parser files found in parsers/ directory and console sync was skipped "
|
||||
f"({fetch_result['skipped']}). "
|
||||
"Add SDL_CONFIG_READ_KEY in Settings (needs 'Manage config files' permission) "
|
||||
"or upload a parser file manually."
|
||||
)
|
||||
if not entries:
|
||||
raise HTTPException(
|
||||
422,
|
||||
"No parser files found in parsers/ directory. "
|
||||
"Use 'Load SDL Parsers via MCP' in Claude Code to populate it, "
|
||||
"or upload a parser file manually."
|
||||
"No parser files found in parsers/ directory after console sync. "
|
||||
"Check SDL_CONFIG_READ_KEY permissions ('Manage config files' required)."
|
||||
)
|
||||
|
||||
loaded = []
|
||||
@@ -258,7 +358,12 @@ async def load_parsers_from_sdl(db: Session = Depends(get_db)):
|
||||
errors.append({"parser": entry.name, "error": str(e)})
|
||||
|
||||
db.commit()
|
||||
return {"loaded": len(loaded), "parsers": loaded, "errors": errors}
|
||||
return {
|
||||
"loaded": len(loaded),
|
||||
"parsers": loaded,
|
||||
"errors": errors,
|
||||
"console_fetch": fetch_result,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/upload-parser")
|
||||
@@ -329,6 +434,9 @@ _S1_NATIVE_SOURCES = {
|
||||
"SentinelOne Ranger AD",
|
||||
}
|
||||
|
||||
# Cached count of events with no dataSource.name — updated on each sync
|
||||
_unlabelled_event_count: int = -1 # -1 = not yet queried
|
||||
|
||||
|
||||
@router.post("/sync-sources")
|
||||
async def sync_sources(days: int = 7, db: Session = Depends(get_db)):
|
||||
@@ -378,28 +486,34 @@ async def sync_sources(days: int = 7, db: Session = Depends(get_db)):
|
||||
parser_detected=parsed_by_source.get(name, 0),
|
||||
))
|
||||
seen += 1
|
||||
|
||||
db.commit()
|
||||
return {"synced": seen, "sources": [r["dataSource.name"] for r in rows if r.get("dataSource.name") and r["dataSource.name"] not in _S1_NATIVE_SOURCES]}
|
||||
synced_names = [r["dataSource.name"] for r in rows if r.get("dataSource.name") and r["dataSource.name"] not in _S1_NATIVE_SOURCES]
|
||||
return {"synced": seen, "sources": synced_names}
|
||||
|
||||
|
||||
def _build_parser_ds_index() -> dict[str, dict]:
|
||||
def _build_parser_ds_index() -> tuple[dict[str, dict], list[dict]]:
|
||||
"""
|
||||
Read all parser files from /app/parsers/ and build an index:
|
||||
dataSource.name (exact, from parser attributes) → {parser_name, format_type}
|
||||
Read all parser files from /app/parsers/ and build:
|
||||
- index: dataSource.name → {parser_name, format_type} (complete parsers)
|
||||
- stubs: list of {parser_name} for files with no dataSource.name attribute
|
||||
|
||||
Format type is "grok", "dottedJson", or "custom".
|
||||
Sources with grok/dottedJson parsers are flagged as needing a proper parser.
|
||||
"""
|
||||
import os, re
|
||||
parsers_dir = "/app/parsers"
|
||||
_DS_NAME_RE = re.compile(r'"dataSource\.name"\s*:\s*"([^"]+)"')
|
||||
_DS_NAME_RE = re.compile(r'"?dataSource\.name"?\s*:\s*"([^"]+)"')
|
||||
_FORMAT_TYPE_RE = re.compile(r'"type"\s*:\s*"([^"]+)"')
|
||||
# Only treat a file as a parser if it has a formats section — rules out dashboards/saved-searches
|
||||
_HAS_FORMATS_RE = re.compile(r'\bformats\s*:', re.IGNORECASE)
|
||||
|
||||
index: dict[str, dict] = {}
|
||||
stubs: list[dict] = []
|
||||
try:
|
||||
entries = [e for e in os.scandir(parsers_dir) if e.is_file() and not e.name.startswith(".")]
|
||||
except FileNotFoundError:
|
||||
return index
|
||||
return index, stubs
|
||||
|
||||
for entry in entries:
|
||||
try:
|
||||
@@ -408,9 +522,15 @@ def _build_parser_ds_index() -> dict[str, dict]:
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
# Skip files that have no formats section — they're dashboards/queries, not parsers
|
||||
if not _HAS_FORMATS_RE.search(content):
|
||||
continue
|
||||
|
||||
# Extract dataSource.name (may appear multiple times — take first)
|
||||
ds_match = _DS_NAME_RE.search(content)
|
||||
if not ds_match:
|
||||
# Has formats but no dataSource.name — genuine stub parser
|
||||
stubs.append({"parser_name": entry.name})
|
||||
continue
|
||||
ds_name = ds_match.group(1).strip()
|
||||
|
||||
@@ -425,7 +545,7 @@ def _build_parser_ds_index() -> dict[str, dict]:
|
||||
|
||||
index[ds_name] = {"parser_name": entry.name, "format_type": fmt}
|
||||
|
||||
return index
|
||||
return index, stubs
|
||||
|
||||
|
||||
@router.get("/map")
|
||||
@@ -447,11 +567,20 @@ def get_coverage_map(db: Session = Depends(get_db)):
|
||||
parser_index.setdefault(pf.parser_name, set()).add(pf.field_name)
|
||||
|
||||
# Build dataSource.name → {parser_name, format_type} index from parser files
|
||||
ds_index = _build_parser_ds_index()
|
||||
ds_index, stub_parsers = _build_parser_ds_index()
|
||||
|
||||
def _normalize(s: str) -> str:
|
||||
return s.lower().replace(" ", "").replace("-", "").replace("_", "").replace(".", "")
|
||||
|
||||
def _find_stub_match(source_name: str) -> dict | None:
|
||||
"""Return stub parser info if a stub filename fuzzy-matches this source name."""
|
||||
sn = _normalize(source_name)
|
||||
for stub in stub_parsers:
|
||||
fn = _normalize(stub["parser_name"])
|
||||
if fn in sn or sn in fn:
|
||||
return stub
|
||||
return None
|
||||
|
||||
def _find_parser_info(source_name: str) -> dict | None:
|
||||
"""
|
||||
Match priority:
|
||||
@@ -514,6 +643,8 @@ def get_coverage_map(db: Session = Depends(get_db)):
|
||||
parser_info = _find_parser_info(src.source_name)
|
||||
parser_in_data = (src.parser_detected or 0) > 0
|
||||
|
||||
stub_info = _find_stub_match(src.source_name) if not parser_info else None
|
||||
|
||||
if parser_info and parser_info["format_type"] == "custom":
|
||||
status = "covered"
|
||||
matched_parser = parser_info["parser_name"]
|
||||
@@ -528,6 +659,12 @@ def get_coverage_map(db: Session = Depends(get_db)):
|
||||
status = "covered"
|
||||
matched_parser = parser_info["parser_name"] if parser_info else "detected in data"
|
||||
format_type = parser_info["format_type"] if parser_info else "unknown"
|
||||
elif stub_info:
|
||||
# A parser file exists but has no dataSource.name — it's a stub/incomplete
|
||||
status = "stub_parser"
|
||||
matched_parser = stub_info["parser_name"]
|
||||
format_type = None
|
||||
stub_info["suggested_ds_name"] = src.source_name
|
||||
else:
|
||||
status = "parser_needed"
|
||||
matched_parser = None
|
||||
@@ -536,7 +673,7 @@ def get_coverage_map(db: Session = Depends(get_db)):
|
||||
if status == "covered":
|
||||
covered_count += 1
|
||||
else:
|
||||
needed_count += 1
|
||||
needed_count += 1 # stub_parser and parser_needed both count as needing work
|
||||
|
||||
rules_for_src: list = [r for r in rule_by_source.get(src.source_name, []) if r["type"] == "library"]
|
||||
|
||||
@@ -614,6 +751,8 @@ def get_coverage_map(db: Session = Depends(get_db)):
|
||||
"status": status,
|
||||
"parser": matched_parser,
|
||||
"format_type": format_type,
|
||||
"unlabelled": bool(src.unlabelled),
|
||||
"stub_suggested_ds_name": stub_info.get("suggested_ds_name") if stub_info and status == "stub_parser" else None,
|
||||
"parser_fields": len(parser_provides),
|
||||
"parser_detected": src.parser_detected or 0,
|
||||
"rules": rules_for_src,
|
||||
@@ -624,13 +763,20 @@ def get_coverage_map(db: Session = Depends(get_db)):
|
||||
"synced_at": src.synced_at.isoformat() if src.synced_at else None,
|
||||
})
|
||||
|
||||
# Only surface stub parsers that matched an active source with real events —
|
||||
# unmatched stubs with zero events are noise and are suppressed.
|
||||
|
||||
synced_at = active_sources[0].synced_at.isoformat() if active_sources else None
|
||||
|
||||
stub_count = sum(1 for s in sources_out if s["status"] == "stub_parser")
|
||||
|
||||
return {
|
||||
"summary": {
|
||||
"active_sources": len(active_sources),
|
||||
"covered": covered_count,
|
||||
"parser_needed": needed_count,
|
||||
"stub_parsers": stub_count,
|
||||
"unlabelled_events": _unlabelled_event_count,
|
||||
"parsers_loaded": len(parser_index),
|
||||
"rules_loaded": len(rules),
|
||||
},
|
||||
@@ -640,9 +786,20 @@ def get_coverage_map(db: Session = Depends(get_db)):
|
||||
}
|
||||
|
||||
|
||||
@router.get("/stub-parsers")
|
||||
def get_stub_parsers():
|
||||
"""Return all parser files that have a formats: section but no dataSource.name attribute.
|
||||
Used by Parser Quality — Attributes Missing section. Independent of active sources."""
|
||||
_, stubs = _build_parser_ds_index()
|
||||
return {"stubs": stubs, "count": len(stubs)}
|
||||
|
||||
|
||||
@router.delete("/reset")
|
||||
def reset_data(db: Session = Depends(get_db)):
|
||||
db.query(ParsedRule).delete()
|
||||
db.query(ParserField).delete()
|
||||
db.query(ActiveSource).delete()
|
||||
db.commit()
|
||||
global _unlabelled_event_count
|
||||
_unlabelled_event_count = -1
|
||||
return {"cleared": True}
|
||||
|
||||
+146
-28
@@ -38,6 +38,7 @@ class SampleEventsRequest(BaseModel):
|
||||
source: str
|
||||
limit: int = 20
|
||||
hours: int = 1
|
||||
filter_mode: str = "broad" # reserved for future use
|
||||
|
||||
|
||||
class FieldPopulationRequest(BaseModel):
|
||||
@@ -107,21 +108,41 @@ def _flatten_event(event: dict) -> dict:
|
||||
def _extract_format_strings(content: str) -> list[str]:
|
||||
"""
|
||||
Extract SDL format string values from augmented-JSON parser content.
|
||||
Matches: "format": "..." (double-quoted value, supports escaped quotes).
|
||||
Handles both:
|
||||
- quoted keys: "format": "..." (valid JSON)
|
||||
- unquoted keys: format: "..." (SDL augmented-JSON)
|
||||
Skips commented-out lines (// ...).
|
||||
"""
|
||||
pattern = re.compile(r'"format"\s*:\s*"((?:[^"\\]|\\.)*)"')
|
||||
return pattern.findall(content)
|
||||
pattern = re.compile(r'(?<!//)\"?format\"?\s*:\s*"((?:[^"\\]|\\.)*)"')
|
||||
results = []
|
||||
for line in content.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("//"):
|
||||
continue
|
||||
results.extend(pattern.findall(line))
|
||||
return results
|
||||
|
||||
|
||||
def _sdl_format_to_regex(fmt: str) -> tuple[re.Pattern, dict[str, str]]:
|
||||
"""
|
||||
Convert an SDL format string to a compiled Python regex.
|
||||
|
||||
Returns (compiled_pattern, py_group_to_sdl_field) mapping so callers can
|
||||
translate group names back to the original SDL field names.
|
||||
SDL format strings may start with '.*,' to absorb a syslog header. When
|
||||
used with re.search that prefix is redundant AND harmful (it forces a comma
|
||||
before the first named field, which won't exist when the log starts with
|
||||
the field directly). We strip the leading '.*,' so re.search can anchor
|
||||
to the first real field at any position in the line.
|
||||
|
||||
Internal '.*' wildcards (field separators for skipped fields) are kept as
|
||||
non-greedy '.*?' so they don't consume adjacent named-field values.
|
||||
|
||||
Returns (compiled_pattern, py_group_to_sdl_field).
|
||||
Raises re.error if the resulting pattern cannot be compiled.
|
||||
"""
|
||||
# Strip leading/trailing .* wildcards — re.search handles positioning
|
||||
fmt = re.sub(r'^(\.\*,?)+', '', fmt)
|
||||
fmt = re.sub(r'(,?\.\*)+$', '', fmt)
|
||||
|
||||
# Split on $...$ tokens
|
||||
token_pattern = re.compile(r'\$([^$]+)\$')
|
||||
parts = token_pattern.split(fmt)
|
||||
@@ -131,19 +152,25 @@ def _sdl_format_to_regex(fmt: str) -> tuple[re.Pattern, dict[str, str]]:
|
||||
py_group_to_sdl: dict[str, str] = {}
|
||||
seen_groups: dict[str, int] = {}
|
||||
|
||||
def _escape_literal(s: str) -> str:
|
||||
"""Escape literal text but keep internal .* as non-greedy wildcards."""
|
||||
segments = re.split(r'(\.\*)', s)
|
||||
return ''.join(r'.*?' if seg == '.*' else re.escape(seg) for seg in segments)
|
||||
|
||||
for i, part in enumerate(parts):
|
||||
if i % 2 == 0:
|
||||
# Literal text
|
||||
regex_parts.append(re.escape(part))
|
||||
# Literal text (possibly containing .* wildcards)
|
||||
regex_parts.append(_escape_literal(part))
|
||||
else:
|
||||
# Token: either "field.name=PATTERN" or just "field.name"
|
||||
if '=' in part:
|
||||
field_name, pattern = part.split('=', 1)
|
||||
else:
|
||||
field_name = part
|
||||
pattern = r'[^\s]+'
|
||||
# Default: match any non-comma chars (SDL CSV fields)
|
||||
pattern = r'[^,]*'
|
||||
|
||||
# Build a valid Python group name
|
||||
# Build a valid Python named-group identifier
|
||||
safe = re.sub(r'[.\-]', '_', field_name)
|
||||
if safe in seen_groups:
|
||||
seen_groups[safe] += 1
|
||||
@@ -154,7 +181,7 @@ def _sdl_format_to_regex(fmt: str) -> tuple[re.Pattern, dict[str, str]]:
|
||||
py_group_to_sdl[safe] = field_name
|
||||
regex_parts.append(f'(?P<{safe}>{pattern})')
|
||||
|
||||
compiled = re.compile(''.join(regex_parts), re.IGNORECASE)
|
||||
compiled = re.compile(''.join(regex_parts), re.IGNORECASE | re.DOTALL)
|
||||
return compiled, py_group_to_sdl
|
||||
|
||||
|
||||
@@ -162,6 +189,45 @@ def _sdl_format_to_regex(fmt: str) -> tuple[re.Pattern, dict[str, str]]:
|
||||
# Endpoints
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.post("/sample-unlabelled")
|
||||
async def sample_unlabelled(req: SampleEventsRequest):
|
||||
"""Return a sample of events that have no dataSource.name — these need parsers.
|
||||
Also runs a count query so the caller can update the banner with the real total.
|
||||
"""
|
||||
import asyncio
|
||||
from routers import coverage as _coverage
|
||||
|
||||
filter_expr = "!(dataSource.name = *) !(source = 'scalyr')"
|
||||
from_dt, to_dt = _date_range_hours(req.hours)
|
||||
|
||||
sample_result, count_result = await asyncio.gather(
|
||||
s1_client.run_powerquery(f"{filter_expr} | limit {req.limit}", from_dt, to_dt),
|
||||
s1_client.run_powerquery(f"{filter_expr} | group events=count()", from_dt, to_dt, max_count=50_000_000),
|
||||
)
|
||||
|
||||
rows = sample_result if isinstance(sample_result, list) else (sample_result.get("rows") or sample_result.get("events") or [])
|
||||
|
||||
events = [_flatten_event(row) for row in rows]
|
||||
non_empty_keys: set = set()
|
||||
for ev in events:
|
||||
for k, v in ev.items():
|
||||
if v is not None and v != "" and v != "null":
|
||||
non_empty_keys.add(k)
|
||||
events = [{k: v for k, v in ev.items() if k in non_empty_keys} for ev in events]
|
||||
|
||||
count_rows = count_result.get("events", []) if isinstance(count_result, dict) else []
|
||||
total = count_rows[0].get("events", 0) if count_rows else 0
|
||||
_coverage._unlabelled_event_count = total
|
||||
|
||||
return {
|
||||
"events": events,
|
||||
"count": len(events),
|
||||
"total": total,
|
||||
"hours": req.hours,
|
||||
"columns_seen": sorted(non_empty_keys),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/sample-events")
|
||||
async def sample_events(req: SampleEventsRequest):
|
||||
"""Return a sample of raw events from a given data source."""
|
||||
@@ -196,21 +262,39 @@ async def field_population(req: FieldPopulationRequest):
|
||||
events = [_flatten_event(row) for row in rows]
|
||||
|
||||
if not events:
|
||||
raise HTTPException(status_code=404, detail=f"No events found for source '{req.source}' in the last {req.hours} hours.")
|
||||
return {
|
||||
"source": req.source,
|
||||
"total_sampled": 0,
|
||||
"hours": req.hours,
|
||||
"fields": [],
|
||||
"fields_seen_in_sample": [],
|
||||
"message": f"No events found for source '{req.source}' in the last {req.hours} hours.",
|
||||
}
|
||||
|
||||
total = len(events)
|
||||
_empty = {None, "", "null"}
|
||||
_empty_scalars = {None, "", "null"}
|
||||
|
||||
def _is_empty(val):
|
||||
"""Return True if the value counts as unpopulated."""
|
||||
if val is None:
|
||||
return True
|
||||
if isinstance(val, list):
|
||||
return len(val) == 0
|
||||
if isinstance(val, dict):
|
||||
return len(val) == 0
|
||||
return val in _empty_scalars
|
||||
|
||||
# Collect all field names seen across the sample (useful for surfacing what IS there)
|
||||
all_seen_fields = sorted({k for ev in events for k in ev})
|
||||
|
||||
all_seen_fields_set = set(all_seen_fields)
|
||||
|
||||
field_stats = []
|
||||
for field in req.fields:
|
||||
# dataSource.name is always 100% — we filtered by it; Scalyr just doesn't echo it back
|
||||
if field == "dataSource.name":
|
||||
populated = total
|
||||
else:
|
||||
populated = sum(1 for ev in events if ev.get(field) not in _empty)
|
||||
# Skip fields that don't appear anywhere in the sample
|
||||
if field not in all_seen_fields_set:
|
||||
continue
|
||||
populated = sum(1 for ev in events if not _is_empty(ev.get(field)))
|
||||
rate = round((populated / total) * 100, 1)
|
||||
field_stats.append({
|
||||
"field": field,
|
||||
@@ -219,8 +303,8 @@ async def field_population(req: FieldPopulationRequest):
|
||||
"rate": rate,
|
||||
})
|
||||
|
||||
# Sort ascending by rate (worst coverage first)
|
||||
field_stats.sort(key=lambda x: x["rate"])
|
||||
# Sort descending by rate (best coverage first)
|
||||
field_stats.sort(key=lambda x: x["rate"], reverse=True)
|
||||
|
||||
return {
|
||||
"source": req.source,
|
||||
@@ -255,7 +339,10 @@ async def test_parser(req: TestParserRequest):
|
||||
# The regex-based path can't model that — handle it explicitly so users
|
||||
# can test JSON-shaped logs against JSON-mode parsers.
|
||||
log_input = req.log_line.strip()
|
||||
is_json_mode = any("parse=json" in f for f in format_strings) or log_input.startswith("{")
|
||||
# Only enter JSON mode if the log content actually looks like JSON.
|
||||
# Don't force it based on the parser type alone — a JSON-capable parser
|
||||
# should still fall through to regex matching for non-JSON inputs.
|
||||
is_json_mode = log_input.startswith("{") or log_input.startswith("[")
|
||||
if is_json_mode:
|
||||
import json as _json
|
||||
# Support multi-line input (one JSON object per line, or a JSON array)
|
||||
@@ -307,18 +394,24 @@ async def test_parser(req: TestParserRequest):
|
||||
# Use the first payload for the detail table; report totals.
|
||||
payload = payloads[0]
|
||||
extracted = _flatten_dict(payload)
|
||||
# SDL's parse=json puts all keys into unmapped.* namespace first, then
|
||||
# rewrites map unmapped.X -> ocsf.field. Mirror that so rewrites fire.
|
||||
unmapped_aliases = {f"unmapped.{k}": v for k, v in extracted.items()}
|
||||
extracted_with_unmapped = {**extracted, **unmapped_aliases}
|
||||
|
||||
# Apply lightweight rewrites if present (input/output/match/replace blocks).
|
||||
# We only handle simple literal/regex matches with $0 or string replacements;
|
||||
# this is best-effort, intended for quick visual verification.
|
||||
rewrites_applied = []
|
||||
# Handle both quoted keys ("input":) and unquoted keys (input:)
|
||||
rewrite_re = re.compile(
|
||||
r'\{\s*input:\s*"([^"]+)"\s*,\s*output:\s*"([^"]+)"\s*,\s*match:\s*"((?:[^"\\]|\\.)*)"\s*,\s*replace:\s*"((?:[^"\\]|\\.)*)"\s*\}',
|
||||
r'\{\s*"?input"?\s*:\s*"([^"]+)"\s*,\s*"?output"?\s*:\s*"([^"]+)"\s*,\s*"?match"?\s*:\s*"((?:[^"\\]|\\.)*)"\s*,\s*"?replace"?\s*:\s*"((?:[^"\\]|\\.)*)"\s*\}',
|
||||
re.DOTALL,
|
||||
)
|
||||
derived: dict[str, str] = {}
|
||||
for m in rewrite_re.finditer(content):
|
||||
in_field, out_field, match_pat, replace_val = m.group(1), m.group(2), m.group(3), m.group(4)
|
||||
src_val = extracted.get(in_field)
|
||||
src_val = extracted_with_unmapped.get(in_field)
|
||||
if src_val is None:
|
||||
continue
|
||||
try:
|
||||
@@ -359,32 +452,57 @@ async def test_parser(req: TestParserRequest):
|
||||
"showing_payload": 1,
|
||||
}
|
||||
|
||||
# ── Regex format-string path (original) ─────────────────────────────────
|
||||
# ── Regex format-string path ─────────────────────────────────────────────
|
||||
def _try_prefix_match(compiled: re.Pattern, py_to_sdl: dict, log_line: str):
|
||||
"""
|
||||
Try the full pattern; if it doesn't match, progressively shorten from
|
||||
the right (group by group) until we get a match. This handles logs
|
||||
that don't include all the trailing optional fields the parser defines.
|
||||
Returns (match, truncated) or (None, False).
|
||||
"""
|
||||
m = compiled.search(log_line)
|
||||
if m:
|
||||
return m, False
|
||||
|
||||
# Shorten pattern by removing trailing named groups one at a time
|
||||
p = compiled.pattern
|
||||
# Find all (?P<name>...) group end positions (right to left)
|
||||
group_ends = [m2.end() for m2 in re.finditer(r'\(\?P<[^>]+>[^)]*\)', p)]
|
||||
for end in reversed(group_ends[1:]): # keep at least 1 group
|
||||
try:
|
||||
shorter = re.compile(p[:end], re.IGNORECASE | re.DOTALL)
|
||||
m2 = shorter.search(log_line)
|
||||
if m2:
|
||||
return m2, True
|
||||
except re.error:
|
||||
continue
|
||||
return None, False
|
||||
|
||||
for fmt in format_strings:
|
||||
try:
|
||||
compiled, py_to_sdl = _sdl_format_to_regex(fmt)
|
||||
except re.error:
|
||||
# Skip unparseable format strings
|
||||
continue
|
||||
|
||||
match = compiled.search(req.log_line)
|
||||
match, truncated = _try_prefix_match(compiled, py_to_sdl, req.log_line)
|
||||
if match:
|
||||
fields = [
|
||||
{"field": py_to_sdl.get(group, group), "value": value}
|
||||
for group, value in match.groupdict().items()
|
||||
if value is not None
|
||||
if value is not None and value != ""
|
||||
]
|
||||
return {
|
||||
"parser_name": req.parser_name,
|
||||
"matched": True,
|
||||
"mode": "regex",
|
||||
"format_matched": fmt,
|
||||
"format_matched": fmt[:120] + ("…" if len(fmt) > 120 else ""),
|
||||
"fields": fields,
|
||||
"note": "Partial match — log has fewer fields than the full parser format" if truncated else None,
|
||||
}
|
||||
|
||||
return {
|
||||
"parser_name": req.parser_name,
|
||||
"matched": False,
|
||||
"message": "No format pattern matched",
|
||||
"message": "No format pattern matched. Check that the log includes the log-type keyword (e.g. TRAFFIC, THREAT) and enough comma-separated fields.",
|
||||
"fields": [],
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ FIELDS = [
|
||||
{"key": "S1_API_TOKEN", "label": "Console API Token", "secret": True, "placeholder": "eyJ..."},
|
||||
{"key": "SDL_XDR_URL", "label": "SDL XDR URL", "secret": False, "placeholder": "https://xdr.us1.sentinelone.net"},
|
||||
{"key": "SDL_LOG_READ_KEY", "label": "SDL Log Read Key", "secret": True, "placeholder": "1DnK0Y4e..."},
|
||||
{"key": "SDL_CONFIG_READ_KEY", "label": "SDL Config Read Key", "secret": True, "placeholder": "Needs 'Manage config files' permission"},
|
||||
{"key": "ANTHROPIC_API_KEY", "label": "Anthropic API Key", "secret": True, "placeholder": "sk-ant-..."},
|
||||
]
|
||||
|
||||
|
||||
+121
-17
@@ -11,6 +11,11 @@ TOKEN = os.environ.get("S1_API_TOKEN", "")
|
||||
SDL_XDR_URL = os.environ.get("SDL_XDR_URL", "https://xdr.us1.sentinelone.net").rstrip("/")
|
||||
SDL_LOG_READ_KEY = os.environ.get("SDL_LOG_READ_KEY", "")
|
||||
|
||||
# SDL Configuration Read Key — used to list/fetch parser files under /logParsers/
|
||||
# (separate from SDL_LOG_READ_KEY which is for querying events only).
|
||||
# Find it in the S1 console: Settings → Integrations → Data Lake API Keys → Configuration Read.
|
||||
SDL_CONFIG_READ_KEY = os.environ.get("SDL_CONFIG_READ_KEY", "")
|
||||
|
||||
# Management Console API uses ApiToken auth
|
||||
HEADERS = {
|
||||
"Authorization": f"ApiToken {TOKEN}",
|
||||
@@ -92,7 +97,7 @@ async def get_library_rules(page_size: int = 100) -> list:
|
||||
return results
|
||||
|
||||
|
||||
async def run_powerquery(query: str, from_date: str, to_date: str) -> dict:
|
||||
async def run_powerquery(query: str, from_date: str, to_date: str, max_count: int = 1000) -> dict:
|
||||
"""
|
||||
Run a PowerQuery against the Singularity Data Lake via the Scalyr XDR API.
|
||||
Uses SDL_XDR_URL + SDL_LOG_READ_KEY (Scalyr readlog token).
|
||||
@@ -109,7 +114,7 @@ async def run_powerquery(query: str, from_date: str, to_date: str) -> dict:
|
||||
"query": query,
|
||||
"startTime": start_ms,
|
||||
"endTime": end_ms,
|
||||
"maxCount": 1000,
|
||||
"maxCount": max_count,
|
||||
}
|
||||
|
||||
async with httpx.AsyncClient(timeout=120) as client:
|
||||
@@ -154,8 +159,47 @@ async def run_powerquery(query: str, from_date: str, to_date: str) -> dict:
|
||||
return {"events": matches}
|
||||
|
||||
|
||||
def _sdl_config_headers() -> dict:
|
||||
"""Auth headers for the SDL Configuration File API (uses POST /api/listFiles,
|
||||
POST /api/getFile, etc.). Falls back to SDL_LOG_READ_KEY if no dedicated
|
||||
Configuration Read key is set — that won't work for all endpoints, but lets
|
||||
callers fail with a meaningful 401 instead of crashing."""
|
||||
key = SDL_CONFIG_READ_KEY or SDL_LOG_READ_KEY
|
||||
return {
|
||||
"Authorization": f"Bearer {key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
async def list_sdl_parsers() -> list[str]:
|
||||
"""List all parser filenames under /logParsers/ in SDL."""
|
||||
"""List parser paths under /logParsers/ via the SDL Configuration File API.
|
||||
|
||||
Requires SDL_CONFIG_READ_KEY (or higher) in .env. The endpoint is
|
||||
POST <SDL_XDR_URL>/api/listFiles with {"pathPrefix": "/logParsers/"}.
|
||||
Returns names without the /logParsers/ prefix, suitable for use as
|
||||
filenames in the local parsers/ directory.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.post(
|
||||
f"{SDL_XDR_URL}/api/listFiles",
|
||||
headers=_sdl_config_headers(),
|
||||
json={"pathPrefix": "/logParsers/"},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
paths = data.get("paths") or data.get("files") or []
|
||||
# Normalize: strip leading /logParsers/ and ignore anything that isn't there
|
||||
names: list[str] = []
|
||||
for p in paths:
|
||||
if isinstance(p, dict):
|
||||
p = p.get("path") or p.get("name") or ""
|
||||
if isinstance(p, str) and p.startswith("/logParsers/"):
|
||||
names.append(p[len("/logParsers/"):])
|
||||
return names
|
||||
|
||||
|
||||
async def list_sdl_parsers_legacy() -> list[str]:
|
||||
"""[Deprecated] Legacy management-console path — kept for reference but unused."""
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.get(
|
||||
f"{BASE_URL}/api/v1/files/logParsers",
|
||||
@@ -170,46 +214,106 @@ async def list_sdl_parsers() -> list[str]:
|
||||
|
||||
|
||||
async def get_sdl_parser(filename: str) -> dict:
|
||||
"""Fetch a single SDL parser file by name."""
|
||||
"""Fetch a single SDL parser file by name via POST /api/getFile.
|
||||
|
||||
Returns the raw SDL response dict, e.g.
|
||||
{"status": "success", "path": "/logParsers/Foo", "content": "...", "version": 3, ...}
|
||||
"""
|
||||
path = filename if filename.startswith("/logParsers/") else f"/logParsers/{filename}"
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
resp = await client.get(
|
||||
f"{BASE_URL}/api/v1/files/logParsers/{filename}",
|
||||
headers=HEADERS,
|
||||
resp = await client.post(
|
||||
f"{SDL_XDR_URL}/api/getFile",
|
||||
headers=_sdl_config_headers(),
|
||||
json={"path": path},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
async def get_account_id() -> str | None:
|
||||
"""Return the first account ID visible to the current token."""
|
||||
"""Return the first account ID visible to the current token.
|
||||
|
||||
Tries /accounts first (works for account-scoped or higher tokens). If that
|
||||
returns 403 (site-scoped token), falls back to /sites and reads accountId
|
||||
from the first site.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
# Path 1: account-scoped token
|
||||
resp = await client.get(
|
||||
f"{BASE_URL}/web/api/v2.1/accounts",
|
||||
headers=HEADERS,
|
||||
params={"limit": 1},
|
||||
)
|
||||
resp.raise_for_status()
|
||||
accounts = resp.json().get("data", [])
|
||||
return str(accounts[0]["id"]) if accounts else None
|
||||
if resp.status_code == 200:
|
||||
accounts = resp.json().get("data", [])
|
||||
if accounts:
|
||||
return str(accounts[0]["id"])
|
||||
# Path 2: site-scoped token — accountId is embedded in sites payload
|
||||
if resp.status_code in (401, 403):
|
||||
sresp = await client.get(
|
||||
f"{BASE_URL}/web/api/v2.1/sites",
|
||||
headers=HEADERS,
|
||||
params={"limit": 1},
|
||||
)
|
||||
if sresp.status_code == 200:
|
||||
data = sresp.json().get("data", {})
|
||||
sites = data.get("sites") if isinstance(data, dict) else data
|
||||
if sites:
|
||||
return str(sites[0].get("accountId") or "") or None
|
||||
return None
|
||||
|
||||
|
||||
async def get_scope_for_platform_rules() -> tuple[str, str] | None:
|
||||
"""Pick the best scope for /detection-library/platform-rules.
|
||||
|
||||
Returns (scopeLevel, scopeId). Tries account first, then site — site-scoped
|
||||
tokens cannot list accounts but CAN query platform-rules with site scope.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=15) as client:
|
||||
# Prefer account scope (broadest)
|
||||
a = await client.get(
|
||||
f"{BASE_URL}/web/api/v2.1/accounts",
|
||||
headers=HEADERS,
|
||||
params={"limit": 1},
|
||||
)
|
||||
if a.status_code == 200:
|
||||
accounts = a.json().get("data", [])
|
||||
if accounts:
|
||||
return ("account", str(accounts[0]["id"]))
|
||||
# Fall back to site scope (site-scoped tokens land here)
|
||||
s = await client.get(
|
||||
f"{BASE_URL}/web/api/v2.1/sites",
|
||||
headers=HEADERS,
|
||||
params={"limit": 1},
|
||||
)
|
||||
if s.status_code == 200:
|
||||
data = s.json().get("data", {})
|
||||
sites = data.get("sites") if isinstance(data, dict) else data
|
||||
if sites:
|
||||
sid = sites[0].get("id")
|
||||
if sid:
|
||||
return ("site", str(sid))
|
||||
return None
|
||||
|
||||
|
||||
async def get_platform_rules(page_size: int = 1000) -> list:
|
||||
"""
|
||||
Fetch all Detection Library platform rules from /detection-library/platform-rules.
|
||||
Requires scopeLevel + scopeId — uses account scope with the first visible account.
|
||||
Returns list of rules, each with a 'sources' list (authoritative data source names).
|
||||
Requires scopeLevel + scopeId. Tries account scope first, then site scope so
|
||||
site-scoped tokens also work.
|
||||
"""
|
||||
account_id = await get_account_id()
|
||||
if not account_id:
|
||||
scope = await get_scope_for_platform_rules()
|
||||
if not scope:
|
||||
return []
|
||||
scope_level, scope_id = scope
|
||||
|
||||
all_rules: list = []
|
||||
cursor: str = ""
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
while True:
|
||||
params: dict = {
|
||||
"scopeLevel": "account",
|
||||
"scopeId": account_id,
|
||||
"scopeLevel": scope_level,
|
||||
"scopeId": scope_id,
|
||||
"limit": page_size,
|
||||
"cursor": cursor,
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ services:
|
||||
- S1_BASE_URL=${S1_BASE_URL}
|
||||
- SDL_XDR_URL=${SDL_XDR_URL}
|
||||
- SDL_LOG_READ_KEY=${SDL_LOG_READ_KEY}
|
||||
- SDL_CONFIG_READ_KEY=${SDL_CONFIG_READ_KEY}
|
||||
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
||||
- DATABASE_URL=postgresql://siem:siem@db:5432/siem
|
||||
- DETECTIONS_FILE=/app/data/detections.json
|
||||
|
||||
+488
-237
File diff suppressed because it is too large
Load Diff
+949
-21
@@ -1,29 +1,957 @@
|
||||
{
|
||||
"attributes": {
|
||||
"dataSource.vendor": "AWS",
|
||||
"dataSource.name": "AWS Web Application Firewall",
|
||||
"dataSource.category": "web_security"
|
||||
"dataSource.name": "AWS CloudTrail",
|
||||
"dataSource.category": "security",
|
||||
"metadata.product.vendor_name": "AWS",
|
||||
"metadata.product.name": "AWS CloudTrail",
|
||||
"metadata.version": "1.0.0"
|
||||
},
|
||||
"formats": [
|
||||
{
|
||||
"id": "aws_waf_json",
|
||||
"format": ".*${parse=json}$",
|
||||
"rewrites": [
|
||||
{ "input": "timestamp", "output": "time", "match": ".*", "replace": "$0" },
|
||||
{ "input": "httpRequest.clientIp", "output": "src_endpoint.ip", "match": ".*", "replace": "$0" },
|
||||
{ "input": "action", "output": "disposition", "match": ".*", "replace": "$0" },
|
||||
{ "input": "httpRequest.uri", "output": "http_request.url.text", "match": ".*", "replace": "$0" },
|
||||
{ "input": "httpRequest.country", "output": "src_endpoint.location.country", "match": ".*", "replace": "$0" },
|
||||
{ "input": "httpRequest.httpMethod", "output": "http_request.http_method", "match": ".*", "replace": "$0" },
|
||||
{ "input": "webaclId", "output": "firewall_rule.uid", "match": ".*", "replace": "$0" },
|
||||
{ "input": "ruleGroupId", "output": "firewall_rule.name", "match": ".*", "replace": "$0" },
|
||||
{ "input": "terminatingRuleType", "output": "firewall_rule.type", "match": ".*", "replace": "$0" },
|
||||
{ "input": "httpRequest.httpVersion", "output": "http_request.version", "match": ".*", "replace": "$0" },
|
||||
{ "input": "httpRequest.args", "output": "http_request.url.query_string", "match": ".*", "replace": "$0" },
|
||||
{ "input": "requestId", "output": "http_request.uid", "match": ".*", "replace": "$0" },
|
||||
{ "input": "httpRequest.headers", "output": "http_request.http_headers", "match": ".*", "replace": "$0" }
|
||||
],
|
||||
"halt": true
|
||||
"format": "${parse=gron}$",
|
||||
"skipNumericConversion": true
|
||||
}
|
||||
]
|
||||
],
|
||||
"mappings": {
|
||||
"version": 1,
|
||||
"mappings": [
|
||||
{
|
||||
"predicate": "eventCategory matches '.*'",
|
||||
"transformations": [
|
||||
{
|
||||
"constant": {
|
||||
"field": "$s1_tmp.predicate_0",
|
||||
"value": true,
|
||||
"predicate": "userIdentity.arn matches '.*'"
|
||||
}
|
||||
}, {
|
||||
"rename_tree": {
|
||||
"from": "",
|
||||
"to": "unmapped"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "message",
|
||||
"from": "unmapped.message"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.message"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "class_uid",
|
||||
"value": 4002
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "metadata.product.name",
|
||||
"value": "AWS CloudTrail"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "metadata.product.vendor_name",
|
||||
"value": "AWS"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "metadata.version",
|
||||
"value": "1.0.0-rc3"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "category_name",
|
||||
"value": "Network Activity"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "category_uid",
|
||||
"value": 4
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "class_uid",
|
||||
"value": 4002
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "class_name",
|
||||
"value": "HTTP Activity"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "metadata.product.name",
|
||||
"value": "CloudTrail"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "metadata.product.vendor_name",
|
||||
"value": "AWS"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "metadata.version",
|
||||
"value": "1.0.0-rc3"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "type_name",
|
||||
"value": "HTTP Activity: Other"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "type_uid",
|
||||
"value": 400299
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "activity_id",
|
||||
"value": 99
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "severity_id",
|
||||
"value": 99
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "status_id",
|
||||
"value": 99
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "status",
|
||||
"value": "Other"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "dataSource.vendor",
|
||||
"value": "AWS"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "dataSource.name",
|
||||
"value": "CloudTrail"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "dataSource.category",
|
||||
"value": "security"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "observables[0].type_id",
|
||||
"value": 2
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "observables[0].type",
|
||||
"value": "IP Address"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "observables[0].name",
|
||||
"value": "src_endpoint.ip"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "observables[1].type_id",
|
||||
"value": 99,
|
||||
"predicate": "unmapped.$s1_tmp.predicate_0 == true"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "observables[1].type",
|
||||
"value": "Other",
|
||||
"predicate": "unmapped.$s1_tmp.predicate_0 == true"
|
||||
}
|
||||
}, {
|
||||
"constant": {
|
||||
"field": "observables[1].name",
|
||||
"value": "unmapped.userIdentity.arn",
|
||||
"predicate": "unmapped.$s1_tmp.predicate_0 == true"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "cloud.region",
|
||||
"from": "unmapped.awsRegion"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "metadata.product.feature.name",
|
||||
"from": "unmapped.eventCategory"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "metadata.uid",
|
||||
"from": "unmapped.eventID"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.eventName",
|
||||
"from": "unmapped.eventName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.service.name",
|
||||
"from": "unmapped.eventSource"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "metadata.original_time",
|
||||
"from": "unmapped.eventTime"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.eventType",
|
||||
"from": "unmapped.eventType"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "metadata.product.version",
|
||||
"from": "unmapped.eventVersion"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.managementEvent",
|
||||
"from": "unmapped.managementEvent"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.readOnly",
|
||||
"from": "unmapped.readOnly"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "cloud.account.uid",
|
||||
"from": "unmapped.recipientAccountId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.request.uid",
|
||||
"from": "unmapped.requestID"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "duration",
|
||||
"from": "unmapped.requestParameters.durationSeconds"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.roleArn",
|
||||
"from": "unmapped.requestParameters.roleArn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.roleSessionName",
|
||||
"from": "unmapped.requestParameters.roleSessionName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.request.uid",
|
||||
"from": "unmapped.requestParameters.externalId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "resource.account.uid[*]",
|
||||
"from": "unmapped.resources[*].accountId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "resource.type[*]",
|
||||
"from": "unmapped.resources[*].type"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "resource.uid[*]",
|
||||
"from": "unmapped.resources[*].ARN"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.responseElements.assumedRoleUser.assumedRoleId",
|
||||
"from": "unmapped.responseElements.assumedRoleUser.assumedRoleId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.responseElements.assumedRoleUser.arn",
|
||||
"from": "unmapped.responseElements.assumedRoleUser.arn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.session.credential_uid",
|
||||
"from": "unmapped.responseElements.credentials.accessKeyId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.responseElements.credentials.sessionToken",
|
||||
"from": "unmapped.responseElements.credentials.sessionToken"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.session.expiration_time",
|
||||
"from": "unmapped.responseElements.credentials.expiration"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.responseElements.sourceIdentity",
|
||||
"from": "unmapped.responseElements.sourceIdentity"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.sharedEventID",
|
||||
"from": "unmapped.sharedEventID"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "src_endpoint.ip",
|
||||
"from": "unmapped.sourceIPAddress"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "tls.version",
|
||||
"from": "unmapped.tlsDetails.tlsVersion"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "tls.cipher",
|
||||
"from": "unmapped.tlsDetails.cipherSuite"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.tlsDetails.clientProvidedHostHeader",
|
||||
"from": "unmapped.tlsDetails.clientProvidedHostHeader"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "http_request.user_agent",
|
||||
"from": "unmapped.userAgent"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.user.account.uid",
|
||||
"from": "unmapped.userIdentity.accountId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.user.uid",
|
||||
"from": "unmapped.userIdentity.principalId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.user.type",
|
||||
"from": "unmapped.userIdentity.type"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.additionalEventData.SignatureVersion",
|
||||
"from": "unmapped.additionalEventData.SignatureVersion"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.additionalEventData.CipherSuite",
|
||||
"from": "unmapped.additionalEventData.CipherSuite"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.additionalEventData.bytesTransferredIn",
|
||||
"from": "unmapped.additionalEventData.bytesTransferredIn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.additionalEventData.AuthenticationMethod",
|
||||
"from": "unmapped.additionalEventData.AuthenticationMethod"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "resources.uid",
|
||||
"from": "unmapped.additionalEventData.x-amz-id-2"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.additionalEventData.bytesTransferredOut",
|
||||
"from": "unmapped.additionalEventData.bytesTransferredOut"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "resources.name",
|
||||
"from": "unmapped.requestParameters.bucketName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "src_endpoint.hostname",
|
||||
"from": "unmapped.requestParameters.Host"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.acl",
|
||||
"from": "unmapped.requestParameters.acl"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.invoked_by",
|
||||
"from": "unmapped.userIdentity.invokedBy"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.keySpec",
|
||||
"from": "unmapped.requestParameters.keySpec"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.keyId",
|
||||
"from": "unmapped.requestParameters.keyId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.encryptionContext.aws:cloudtrail:arn",
|
||||
"from": "unmapped.requestParameters.encryptionContext.aws:cloudtrail:arn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.encryptionContext.aws:s3:arn",
|
||||
"from": "unmapped.requestParameters.encryptionContext.aws:s3:arn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.agentVersion",
|
||||
"from": "unmapped.requestParameters.agentVersion"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.agentStatus",
|
||||
"from": "unmapped.requestParameters.agentStatus"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.platformType",
|
||||
"from": "unmapped.requestParameters.platformType"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.platformName",
|
||||
"from": "unmapped.requestParameters.platformName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.platformVersion",
|
||||
"from": "unmapped.requestParameters.platformVersion"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.iPAddress",
|
||||
"from": "unmapped.requestParameters.iPAddress"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.computerName",
|
||||
"from": "unmapped.requestParameters.computerName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.agentName",
|
||||
"from": "unmapped.requestParameters.agentName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "src_endpoint.instance_uid",
|
||||
"from": "unmapped.requestParameters.instanceId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.maxResults",
|
||||
"from": "unmapped.requestParameters.maxResults"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "cloud.zone",
|
||||
"from": "unmapped.requestParameters.availabilityZone"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.availabilityZoneId",
|
||||
"from": "unmapped.requestParameters.availabilityZoneId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.user.credential_uid",
|
||||
"from": "unmapped.userIdentity.accessKeyId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.userIdentity.sessionContext.webIdFederationData",
|
||||
"from": "unmapped.userIdentity.sessionContext.webIdFederationData"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.user.name",
|
||||
"from": "unmapped.userIdentity.sessionContext.sessionIssuer.type"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.session.uid",
|
||||
"from": "unmapped.userIdentity.sessionContext.sessionIssuer.principalId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.session.issuer",
|
||||
"from": "unmapped.userIdentity.sessionContext.sessionIssuer.arn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.user.account.uid",
|
||||
"from": "unmapped.userIdentity.sessionContext.sessionIssuer.accountId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.session.issuer",
|
||||
"from": "unmapped.userIdentity.sessionContext.sessionIssuer.userName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.userIdentity.sessionContext.ec2RoleDelivery",
|
||||
"from": "unmapped.userIdentity.sessionContext.ec2RoleDelivery"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.session.created_time",
|
||||
"from": "unmapped.userIdentity.sessionContext.attributes.creationDate"
|
||||
}
|
||||
}, {
|
||||
"cast": {
|
||||
"field": "actor.session.created_time",
|
||||
"type": "iso8601TimestampToEpochSec"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.userIdentity.sessionContext.attributes.mfaAuthenticated",
|
||||
"from": "unmapped.userIdentity.sessionContext.attributes.mfaAuthenticated"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.userIdentity.arn",
|
||||
"from": "unmapped.userIdentity.arn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "actor.user.name",
|
||||
"from": "unmapped.userIdentity.userName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.response.error",
|
||||
"from": "unmapped.errorCode"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.response.error_message",
|
||||
"from": "unmapped.errorMessage"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.edgeDeviceDetails",
|
||||
"from": "unmapped.edgeDeviceDetails"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.sessionCredentialFromConsole",
|
||||
"from": "unmapped.sessionCredentialFromConsole"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "src_endpoint.uid",
|
||||
"from": "unmapped.vpcEndpointId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.serviceEventDetails",
|
||||
"from": "unmapped.serviceEventDetails"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.version",
|
||||
"from": "unmapped.apiVersion"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.policy",
|
||||
"from": "unmapped.requestParameters.policy"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.encryption",
|
||||
"from": "unmapped.requestParameters.encryption"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.publicAccessBlock",
|
||||
"from": "unmapped.requestParameters.publicAccessBlock"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.topicArn",
|
||||
"from": "unmapped.requestParameters.topicArn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.detectorId",
|
||||
"from": "unmapped.requestParameters.detectorId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.website",
|
||||
"from": "unmapped.requestParameters.website"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.nextToken",
|
||||
"from": "unmapped.requestParameters.nextToken"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.certificateArn",
|
||||
"from": "unmapped.requestParameters.certificateArn"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.ownershipControls",
|
||||
"from": "unmapped.requestParameters.ownershipControls"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.maxRecords",
|
||||
"from": "unmapped.requestParameters.maxRecords"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.DescribeInstanceTypesRequest.NextToken",
|
||||
"from": "unmapped.requestParameters.DescribeInstanceTypesRequest.NextToken"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.DescribeInstanceTypesRequest.MaxResults",
|
||||
"from": "unmapped.requestParameters.DescribeInstanceTypesRequest.MaxResults"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.resourceIds",
|
||||
"from": "unmapped.requestParameters.resourceIds"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.dBSnapshotIdentifier",
|
||||
"from": "unmapped.requestParameters.dBSnapshotIdentifier"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.includeShared",
|
||||
"from": "unmapped.requestParameters.includeShared"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.includePublic",
|
||||
"from": "unmapped.requestParameters.includePublic"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.resourceIdList",
|
||||
"from": "unmapped.requestParameters.resourceIdList"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.logGroupName",
|
||||
"from": "unmapped.requestParameters.logGroupName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.replication",
|
||||
"from": "unmapped.requestParameters.replication"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.versioning",
|
||||
"from": "unmapped.requestParameters.versioning"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.tagging",
|
||||
"from": "unmapped.requestParameters.tagging"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.logging",
|
||||
"from": "unmapped.requestParameters.logging"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.workGroup",
|
||||
"from": "unmapped.requestParameters.workGroup"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.clusterStates",
|
||||
"from": "unmapped.requestParameters.clusterStates"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.DescribeVpcEndpointsRequest",
|
||||
"from": "unmapped.requestParameters.DescribeVpcEndpointsRequest"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.GetEbsDefaultKmsKeyIdRequest",
|
||||
"from": "unmapped.requestParameters.GetEbsDefaultKmsKeyIdRequest"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.DescribeVpcEndpointServiceConfigurationsRequest",
|
||||
"from": "unmapped.requestParameters.DescribeVpcEndpointServiceConfigurationsRequest"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.requestParameters.DescribeTransitGatewaysRequest",
|
||||
"from": "unmapped.requestParameters.DescribeTransitGatewaysRequest"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.request.uid",
|
||||
"from": "unmapped.requestParameters.requestContext.awsAccountId"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.insightDetails.state",
|
||||
"from": "unmapped.insightDetails.state"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "api.service.name",
|
||||
"from": "unmapped.insightDetails.eventSource"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.insightDetails.eventName",
|
||||
"from": "unmapped.insightDetails.eventName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.insightDetails.insightType",
|
||||
"from": "unmapped.insightDetails.insightType"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.insightDetails.insightContext.statistics.baseline.average",
|
||||
"from": "unmapped.insightDetails.insightContext.statistics.baseline.average"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "unmapped.insightDetails.insightContext.statistics.insight.average",
|
||||
"from": "unmapped.insightDetails.insightContext.statistics.insight.average"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "duration",
|
||||
"from": "unmapped.insightDetails.insightContext.statistics.insightDuration"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "event.type",
|
||||
"from": "unmapped.eventName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "activity_name",
|
||||
"from": "unmapped.eventName"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "observables[0].value",
|
||||
"from": "unmapped.sourceIPAddress"
|
||||
}
|
||||
}, {
|
||||
"copy": {
|
||||
"to": "observables[1].value",
|
||||
"from": "unmapped.userIdentity.arn",
|
||||
"predicate": "unmapped.$s1_tmp.predicate_0 == true"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.awsRegion"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.eventCategory"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.eventID"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.eventSource"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.eventTime"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.eventVersion"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.recipientAccountId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestID"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestParameters.durationSeconds"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestParameters.externalId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.resources[*].accountId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.resources[*].type"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.resources[*].ARN"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.responseElements.credentials.accessKeyId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.responseElements.credentials.expiration"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.sourceIPAddress"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.tlsDetails.tlsVersion"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.tlsDetails.cipherSuite"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userAgent"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.accountId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.principalId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.type"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.additionalEventData.x-amz-id-2"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestParameters.bucketName"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestParameters.Host"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.invokedBy"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestParameters.instanceId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestParameters.availabilityZone"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.accessKeyId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.sessionContext.sessionIssuer.type"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.sessionContext.sessionIssuer.principalId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.sessionContext.sessionIssuer.arn"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.sessionContext.sessionIssuer.accountId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.sessionContext.sessionIssuer.userName"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.sessionContext.attributes.creationDate"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.userIdentity.userName"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.errorCode"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.errorMessage"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.vpcEndpointId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.apiVersion"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.requestParameters.requestContext.awsAccountId"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.insightDetails.eventSource"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.insightDetails.insightContext.statistics.insightDuration"
|
||||
}
|
||||
}, {
|
||||
"drop": {
|
||||
"field": "unmapped.$s1_tmp.predicate_0"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+158
-19
@@ -2,14 +2,27 @@
|
||||
"attributes": {
|
||||
"dataSource.vendor": "Cisco",
|
||||
"dataSource.name": "Cisco Duo Security",
|
||||
"dataSource.category": "security"
|
||||
"dataSource.category": "security",
|
||||
"metadata.product.vendor_name": "Cisco",
|
||||
"metadata.product.name": "Cisco Duo Security",
|
||||
"metadata.version": "1.0.0"
|
||||
},
|
||||
"formats": [
|
||||
{
|
||||
"format": "$unmapped.{parse=json}$",
|
||||
"rewrites": [
|
||||
{ "input": "unmapped.timestamp", "output": "timestamp", "match": ".*", "replace": "$0" },
|
||||
{ "input": "unmapped.time", "output": "time", "match": ".*", "replace": "$0" }
|
||||
{
|
||||
"input": "unmapped.timestamp",
|
||||
"output": "timestamp",
|
||||
"match": ".*",
|
||||
"replace": "$0"
|
||||
},
|
||||
{
|
||||
"input": "unmapped.time",
|
||||
"output": "time",
|
||||
"match": ".*",
|
||||
"replace": "$0"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
@@ -19,24 +32,150 @@
|
||||
{
|
||||
"predicate": "unmapped.class_uid = '3002'",
|
||||
"transformations": [
|
||||
{ "copy": { "from": "unmapped.activity_id", "to": "activity_id" } },
|
||||
{ "copy": { "from": "unmapped.activity_name", "to": "activity_name" } },
|
||||
{ "copy": { "from": "unmapped.type_uid", "to": "type_uid" } },
|
||||
{ "copy": { "from": "unmapped.severity_id", "to": "severity_id" } },
|
||||
{ "copy": { "from": "unmapped.status_id", "to": "status_id" } },
|
||||
{ "copy": { "from": "unmapped.status", "to": "status" } },
|
||||
{ "copy": { "from": "unmapped.message", "to": "message" } },
|
||||
{ "copy": { "from": "unmapped.user.name", "to": "user.name" } },
|
||||
{ "copy": { "from": "unmapped.user.account_uid", "to": "user.account_uid" } },
|
||||
{ "copy": { "from": "unmapped.user.account_type", "to": "user.account_type" } },
|
||||
{ "copy": { "from": "unmapped.src_endpoint.ip", "to": "src_endpoint.ip" } },
|
||||
{ "copy": { "from": "unmapped.src_endpoint.location.desc", "to": "src_endpoint.location.desc" } },
|
||||
{ "copy": { "from": "unmapped.src_endpoint.location.city", "to": "src_endpoint.location.city" } },
|
||||
{ "copy": { "from": "unmapped.src_endpoint.location.country", "to": "src_endpoint.location.country" } },
|
||||
{ "copy": { "from": "unmapped.auth_protocol", "to": "auth_protocol" } },
|
||||
{ "copy": { "from": "unmapped.mfa_factors", "to": "mfa_factors" } }
|
||||
{
|
||||
"constant": {
|
||||
"value": 3002,
|
||||
"field": "class_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "Authentication",
|
||||
"field": "class_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 3,
|
||||
"field": "category_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "Identity & Access Management",
|
||||
"field": "category_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.activity_id",
|
||||
"to": "activity_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.activity_name",
|
||||
"to": "activity_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.type_uid",
|
||||
"to": "type_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.severity_id",
|
||||
"to": "severity_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.status_id",
|
||||
"to": "status_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.status",
|
||||
"to": "status"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.message",
|
||||
"to": "message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.user.name",
|
||||
"to": "user.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.user.account_uid",
|
||||
"to": "user.account_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.user.account_type",
|
||||
"to": "user.account_type"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.src_endpoint.ip",
|
||||
"to": "src_endpoint.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.src_endpoint.location.desc",
|
||||
"to": "src_endpoint.location.desc"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.src_endpoint.location.city",
|
||||
"to": "src_endpoint.location.city"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.src_endpoint.location.country",
|
||||
"to": "src_endpoint.location.country"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.auth_protocol",
|
||||
"to": "auth_protocol"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.auth_protocol_id",
|
||||
"to": "auth_protocol_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.mfa_factors",
|
||||
"to": "mfa_factors"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"observables": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "user.name",
|
||||
"type": "User"
|
||||
},
|
||||
{
|
||||
"name": "src_endpoint.ip",
|
||||
"type": "IP Address"
|
||||
},
|
||||
{
|
||||
"name": "auth_protocol",
|
||||
"type": "Other"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,363 @@
|
||||
{
|
||||
attributes: {
|
||||
dataset: "Endpoint",
|
||||
"dataSource.name": "CrowdStrike Falcon",
|
||||
"attributes": {
|
||||
"dataSource.vendor": "CrowdStrike",
|
||||
"dataSource.category": "security"
|
||||
}
|
||||
patterns: {
|
||||
keyPattern: "\\w+"
|
||||
lastValuePattern: "[\\w\\s]+"
|
||||
"dataSource.name": "CrowdStrike Endpoint",
|
||||
"dataSource.category": "security",
|
||||
"metadata.product.vendor_name": "CrowdStrike",
|
||||
"metadata.product.name": "CrowdStrike Falcon",
|
||||
"metadata.version": "1.0.0"
|
||||
},
|
||||
formats: [
|
||||
"formats": [
|
||||
{
|
||||
format: "CEF:$version$\\|$deviceVendor$\\|$deviceProduct$\\|$deviceVersion$\\|$signatureID$\\|$name$\\|$severity$\\|$extension$"
|
||||
},
|
||||
{
|
||||
format: ".*[\\s]$_=keyPattern$=$_$ \\w+=",
|
||||
repeat: true
|
||||
},
|
||||
{
|
||||
format: ".*\\s$_=keyPattern$=$_=lastValuePattern$",
|
||||
repeat: true
|
||||
"format": "$unmapped.{parse=json}$",
|
||||
"rewrites": [
|
||||
{
|
||||
"input": "unmapped.timestamp",
|
||||
"output": "timestamp",
|
||||
"match": ".*",
|
||||
"replace": "$0"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
],
|
||||
"mappings": {
|
||||
"version": 1,
|
||||
"mappings": [
|
||||
{
|
||||
"predicate": "true",
|
||||
"transformations": [
|
||||
{
|
||||
"constant": {
|
||||
"value": 1001,
|
||||
"field": "class_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "Process Activity",
|
||||
"field": "class_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "category_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "System Activity",
|
||||
"field": "category_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.timestamp",
|
||||
"to": "time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"replace": {
|
||||
"field": "time",
|
||||
"regexp": "(\\d+)\\d{3}",
|
||||
"replacement": "$1"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.event_id",
|
||||
"to": "metadata.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.name",
|
||||
"to": "message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.event_simpleName",
|
||||
"to": "activity_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.ComputerName",
|
||||
"to": "device.hostname"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.aid",
|
||||
"to": "device.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.aip",
|
||||
"to": "device.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.cid",
|
||||
"to": "device.org.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.UserName",
|
||||
"to": "actor.user.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.FileName",
|
||||
"to": "process.file.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.FilePath",
|
||||
"to": "process.file.path"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.CommandLine",
|
||||
"to": "process.cmd_line"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.ProcessId",
|
||||
"to": "process.pid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.RawProcessId",
|
||||
"to": "process.pid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.ParentProcessId",
|
||||
"to": "process.parent_process.pid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.ParentBaseFileName",
|
||||
"to": "process.parent_process.file.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.SHA256HashData",
|
||||
"to": "process.file.hashes[0].value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.SHA1HashData",
|
||||
"to": "process.file.hashes[1].value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.MD5HashData",
|
||||
"to": "process.file.hashes[2].value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.LocalIP",
|
||||
"to": "src_endpoint.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.LocalPort",
|
||||
"to": "src_endpoint.port"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.RemoteIP",
|
||||
"to": "dst_endpoint.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.RemotePort",
|
||||
"to": "dst_endpoint.port"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.Protocol",
|
||||
"to": "connection_info.protocol_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.RegObjectName",
|
||||
"to": "registry.key"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.RegValueName",
|
||||
"to": "registry.value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.DetectName",
|
||||
"to": "finding.title"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.DetectDescription",
|
||||
"to": "finding.desc"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.Severity",
|
||||
"to": "severity_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.Tactic",
|
||||
"to": "finding.supporting_data.tactic"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.Technique",
|
||||
"to": "finding.supporting_data.technique"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.IOCType",
|
||||
"to": "finding.supporting_data.ioc_type"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.IOCValue",
|
||||
"to": "finding.supporting_data.ioc_value"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.FalconHostLink",
|
||||
"to": "metadata.extensions.falcon_link"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.SensorId",
|
||||
"to": "device.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.ExternalApiType",
|
||||
"to": "metadata.extensions.api_type"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.PatternDisposition",
|
||||
"to": "finding.supporting_data.pattern_disposition"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "SHA256",
|
||||
"field": "process.file.hashes[0].type_id",
|
||||
"predicate": "unmapped.SHA256HashData != \"\""
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "SHA1",
|
||||
"field": "process.file.hashes[1].type_id",
|
||||
"predicate": "unmapped.SHA1HashData != \"\""
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "MD5",
|
||||
"field": "process.file.hashes[2].type_id",
|
||||
"predicate": "unmapped.MD5HashData != \"\""
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "activity_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "status_id"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"observables": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "device.hostname",
|
||||
"type": "Hostname"
|
||||
},
|
||||
{
|
||||
"name": "actor.user.name",
|
||||
"type": "User"
|
||||
},
|
||||
{
|
||||
"name": "process.file.name",
|
||||
"type": "File Name"
|
||||
},
|
||||
{
|
||||
"name": "process.file.path",
|
||||
"type": "File Name"
|
||||
},
|
||||
{
|
||||
"name": "process.file.hashes[0].value",
|
||||
"type": "File Hash"
|
||||
},
|
||||
{
|
||||
"name": "process.file.hashes[1].value",
|
||||
"type": "File Hash"
|
||||
},
|
||||
{
|
||||
"name": "process.file.hashes[2].value",
|
||||
"type": "File Hash"
|
||||
},
|
||||
{
|
||||
"name": "src_endpoint.ip",
|
||||
"type": "IP Address"
|
||||
},
|
||||
{
|
||||
"name": "dst_endpoint.ip",
|
||||
"type": "IP Address"
|
||||
},
|
||||
{
|
||||
"name": "finding.supporting_data.ioc_value",
|
||||
"type": "Other"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,33 +1,327 @@
|
||||
{
|
||||
attributes: {
|
||||
"dataSource.category": "security",
|
||||
"attributes": {
|
||||
"dataSource.vendor": "Microsoft",
|
||||
"dataSource.name": "Azure AD",
|
||||
"dataSource.vendor": "Azure"
|
||||
"dataSource.category": "security",
|
||||
"metadata.product.vendor_name": "Microsoft",
|
||||
"metadata.product.name": "Azure Active Directory",
|
||||
"metadata.version": "1.0.0"
|
||||
},
|
||||
formats: [
|
||||
"formats": [
|
||||
{
|
||||
format: ".*${parse=json}{attrBlacklist=(targetResources)}$"
|
||||
rewrites: [
|
||||
{ input: "activityDateTime", output: "security_finding.time_dt", match: ".*", replace: "$0" },
|
||||
{ input: "activityDisplayName", output: "security_finding.activity_name", match: ".*", replace: "$0" },
|
||||
{ input: "category", output: "security_finding.category_name", match: ".*", replace: "$0" },
|
||||
{ input: "correlationId", output: "metadata.correlation_uid", match: ".*", replace: "$0" },
|
||||
{ input: "id", output: "security_finding.activity_id", match: ".*", replace: "$0" },
|
||||
{ input: "initiatedByUserId", output: "user.account_uid", match: ".*", replace: "$0" },
|
||||
{ input: "initiatedByUserIpAddress", output: "user.ip", match: ".*", replace: "$0" },
|
||||
{ input: "initiatedByUserUserPrincipalName", output: "user.name", match: ".*", replace: "$0" },
|
||||
{ input: "operationType", output: "security_finding.type_name", match: ".*", replace: "$0" },
|
||||
{ input: "result", output: "security_finding.result", match: ".*", replace: "$0" },
|
||||
{ input: "resultReason", output: "security_finding.result_reason", match: ".*", replace: "$0" }
|
||||
]
|
||||
}, {
|
||||
format: ".*targetResources\":..$targetResources.{parse=json}$"
|
||||
rewrites: [
|
||||
{ input: "targetResources.displayName", output: "target.name", match: ".*", replace: "$0" },
|
||||
{ input: "targetResources.id", output: "target.id", match: ".*", replace: "$0" },
|
||||
{ input: "targetResources.type", output: "target.type", match: ".*", replace: "$0" },
|
||||
{ input: "targetResources.userPrincipalName", output: "target.userName", match: ".*", replace: "$0" }
|
||||
"format": "$unmapped.{parse=json}$",
|
||||
"rewrites": [
|
||||
{
|
||||
"input": "unmapped.time",
|
||||
"output": "timestamp",
|
||||
"match": ".*",
|
||||
"replace": "$0"
|
||||
},
|
||||
{
|
||||
"input": "unmapped.activityDateTime",
|
||||
"output": "timestamp",
|
||||
"match": ".*",
|
||||
"replace": "$0"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
],
|
||||
"mappings": {
|
||||
"version": 1,
|
||||
"mappings": [
|
||||
{
|
||||
"predicate": "true",
|
||||
"transformations": [
|
||||
{
|
||||
"constant": {
|
||||
"value": 3002,
|
||||
"field": "class_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "Authentication",
|
||||
"field": "class_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 3,
|
||||
"field": "category_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "Identity & Access Management",
|
||||
"field": "category_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.time",
|
||||
"to": "time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.activityDateTime",
|
||||
"to": "time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cast": {
|
||||
"field": "time",
|
||||
"type": "iso8601TimestampToEpochSec"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.id",
|
||||
"to": "metadata.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.activityDisplayName",
|
||||
"to": "message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.activity",
|
||||
"to": "activity_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.userPrincipalName",
|
||||
"to": "user.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.displayName",
|
||||
"to": "user.full_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.userId",
|
||||
"to": "user.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.ipAddress",
|
||||
"to": "src_endpoint.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.clientAppUsed",
|
||||
"to": "http_request.user_agent"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.userAgent",
|
||||
"to": "http_request.user_agent"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.location.city",
|
||||
"to": "src_endpoint.location.city"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.location.state",
|
||||
"to": "src_endpoint.location.region"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.location.countryOrRegion",
|
||||
"to": "src_endpoint.location.country"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.location.geoCoordinates.latitude",
|
||||
"to": "src_endpoint.location.coordinates[0]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.location.geoCoordinates.longitude",
|
||||
"to": "src_endpoint.location.coordinates[1]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.result",
|
||||
"to": "status"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.resultReason",
|
||||
"to": "status_detail"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.operationType",
|
||||
"to": "activity_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.category",
|
||||
"to": "category_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.correlationId",
|
||||
"to": "metadata.correlation_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.resourceDisplayName",
|
||||
"to": "dst_endpoint.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.resourceId",
|
||||
"to": "dst_endpoint.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.targetResources[0].displayName",
|
||||
"to": "dst_endpoint.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.targetResources[0].id",
|
||||
"to": "dst_endpoint.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.targetResources[0].userPrincipalName",
|
||||
"to": "dst_endpoint.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.authenticationDetails[0].authenticationMethod",
|
||||
"to": "auth_protocol"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.authenticationDetails[0].succeeded",
|
||||
"to": "status"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.conditionalAccessStatus",
|
||||
"to": "metadata.extensions.conditional_access_status"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.isInteractive",
|
||||
"to": "metadata.extensions.is_interactive"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.riskLevel",
|
||||
"to": "risk_level"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.riskState",
|
||||
"to": "risk_level_id"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "activity_id",
|
||||
"predicate": "unmapped.result = 'success'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 2,
|
||||
"field": "activity_id",
|
||||
"predicate": "unmapped.result = 'failure'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "severity_id",
|
||||
"predicate": "unmapped.result = 'success'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 3,
|
||||
"field": "severity_id",
|
||||
"predicate": "unmapped.result = 'failure'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "status_id",
|
||||
"predicate": "unmapped.result = 'success'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 2,
|
||||
"field": "status_id",
|
||||
"predicate": "unmapped.result = 'failure'"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"observables": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "user.name",
|
||||
"type": "User"
|
||||
},
|
||||
{
|
||||
"name": "src_endpoint.ip",
|
||||
"type": "IP Address"
|
||||
},
|
||||
{
|
||||
"name": "user.uid",
|
||||
"type": "User"
|
||||
},
|
||||
{
|
||||
"name": "dst_endpoint.name",
|
||||
"type": "Other"
|
||||
},
|
||||
{
|
||||
"name": "metadata.correlation_uid",
|
||||
"type": "Other"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -1,46 +1,410 @@
|
||||
{
|
||||
attributes: {
|
||||
"dataSource.category": "security",
|
||||
"dataSource.name": "Netskope",
|
||||
"dataSource.vendor": "Netskope"
|
||||
},
|
||||
formats: [
|
||||
{
|
||||
format: ".*${parse=json}$"
|
||||
rewrites: [
|
||||
{ input: "_category_id", output: "security_finding.category_uid", match: ".*", replace: "$0" },
|
||||
{ input: "_correlation_id", output: "metadata.correlation_uid", match: ".*", replace: "$0" },
|
||||
{ input: "_detection_name", output: "detection.name", match: ".*", replace: "$0" },
|
||||
{ input: "_event_id", output: "security_finding.activity_id", match: ".*", replace: "$0" },
|
||||
{ input: "_id", output: "security_finding.type_uid", match: ".*", replace: "$0" },
|
||||
{ input: "_nshostname", output: "network_endpoint.sender_hostname", match: ".*", replace: "$0" },
|
||||
{ input: "_resource_name", output: "resource.name", match: ".*", replace: "$0" },
|
||||
{ input: "account_name", output: "account.name", match: ".*", replace: "$0" },
|
||||
{ input: "action", output: "security_finding.action", match: ".*", replace: "$0" },
|
||||
{ input: "alert_id", output: "alert.uid", match: ".*", replace: "$0" },
|
||||
{ input: "alert_name", output: "alert.name", match: ".*", replace: "$0" },
|
||||
{ input: "alert_type", output: "event.type", match: ".*", replace: "$0" },
|
||||
{ input: "device", output: "device.name", match: ".*", replace: "$0" },
|
||||
{ input: "dlp_file", output: "dlp.file_name", match: ".*", replace: "$0" },
|
||||
{ input: "dlp_incident_id", output: "dlp.incident_id", match: ".*", replace: "$0" },
|
||||
{ input: "dlp_rule", output: "dlp.rule", match: ".*", replace: "$0" },
|
||||
{ input: "dstip", output: "dst.ip.address", match: ".*", replace: "$0" },
|
||||
{ input: "file_name", output: "file.name", match: ".*", replace: "$0" },
|
||||
{ input: "file_size", output: "file.size", match: ".*", replace: "$0" },
|
||||
{ input: "file_type", output: "file.type", match: ".*", replace: "$0" },
|
||||
{ input: "hostname", output: "device.hostname", match: ".*", replace: "$0" },
|
||||
{ input: "malware_name", output: "malware.name", match: ".*", replace: "$0" },
|
||||
{ input: "md5", output: "file.md5", match: ".*", replace: "$0" },
|
||||
{ input: "os", output: "os.name", match: ".*", replace: "$0" },
|
||||
{ input: "policy", output: "policy.name", match: ".*", replace: "$0" },
|
||||
{ input: "policy_id", output: "policy.uid", match: ".*", replace: "$0" },
|
||||
{ input: "protocol", output: "network_connection_info.protocol_name", match: ".*", replace: "$0" },
|
||||
{ input: "srcip", output: "src.ip.address", match: ".*", replace: "$0" },
|
||||
{ input: "url", output: "url.text", match: ".*", replace: "$0" },
|
||||
{ input: "user", output: "user.name", match: ".*", replace: "$0" },
|
||||
{ input: "user_id", output: "user.uid", match: ".*", replace: "$0" },
|
||||
{ input: "userip", output: "user.ip", match: ".*", replace: "$0" }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
attributes: {
|
||||
"dataSource.category": "security",
|
||||
"dataSource.name": "Netskope",
|
||||
"dataSource.vendor": "Netskope"
|
||||
},
|
||||
formats: [
|
||||
{
|
||||
format: ".*${parse=json}{attrWhitelist=(_id|action|activity|device|category|app_name|count|dst_country|dst_region|malware_severity|malware_type|request_id|severity|severity_id|src_country|src_location|src_region|type|_category_id|_category_tags|_correlation_id|_detection_name|_event_id|_nshostname|_resource_name|_service_identifier|account_name|alert_id|alert_name|alert_type|appcategory|breach_date|breach_description|breach_id|breach_score|connection_id|dlp_file|dlp_incident_id|dlp_rule|dlp_rule_count|dst_latitude|dst_longitude|dst_timezone|dst_zipcode|dstip|event_type|file_name|file_size|file_type|hostname|incident_id|instance_id|local_md5|malsite_country|malware_name|matched_username|md5|os|os10|os11|policy|policy_id|protocol|severity_level|severity_level_id|src_latitude|src_longitude|src_time|src_timezone|src_zipcode|srcip|timestamp|transaction_id|true_obj_type|url|user|user_id|userip|userkey|local_sha256)}$"
|
||||
rewrites: [
|
||||
{
|
||||
input: "_category_id",
|
||||
output: "security_finding.category_uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_category_tags",
|
||||
output: "security_finding.category_tags",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_correlation_id",
|
||||
output: "metadata.correlation_uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_detection_name",
|
||||
output: "detection.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_event_id",
|
||||
output: "security_finding.activity_id",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_id",
|
||||
output: "security_finding.type_uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_nshostname",
|
||||
output: "network_endpoint.sender_hostname",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_resource_name",
|
||||
output: "resource.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "_service_identifier",
|
||||
output: "service.identifier",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "account_name",
|
||||
output: "account.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "action",
|
||||
output: "security_finding.action",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "alert_id",
|
||||
output: "alert.uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "alert_name",
|
||||
output: "alert.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "alert_type",
|
||||
output: "event.type",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "appcategory",
|
||||
output: "security_finding.app_category",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "breach_date",
|
||||
output: "breach.date",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "breach_description",
|
||||
output: "breach.desc",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "breach_id",
|
||||
output: "breach.uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "breach_score",
|
||||
output: "breach.score",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "category",
|
||||
output: "security_finding.category_name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "connection_id",
|
||||
output: "security_finding.connection_id",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "device",
|
||||
output: "device.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dlp_file",
|
||||
output: "dlp.file_name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dlp_incident_id",
|
||||
output: "dlp.incident_id",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dlp_rule",
|
||||
output: "dlp.rule",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dlp_rule_count",
|
||||
output: "dlp.rule_count",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dst_latitude",
|
||||
output: "geo_coordinates.dst_latitude",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dst_longitude",
|
||||
output: "geo_coordinates.dst_longitude",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dst_timezone",
|
||||
output: "location.dst_timezone",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dst_zipcode",
|
||||
output: "location.dst_zipcode",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "dstip",
|
||||
output: "dst.ip.address",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "event_type",
|
||||
output: "security_finding.ref_event_name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "file_name",
|
||||
output: "file.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "file_size",
|
||||
output: "file.size",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "file_type",
|
||||
output: "file.type",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "hostname",
|
||||
output: "device.hostname",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "incident_id",
|
||||
output: "security_finding.incident_id",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "instance_id",
|
||||
output: "device.instance_uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "local_md5",
|
||||
output: "file.local_md5",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "malsite_country",
|
||||
output: "security_finding.malsite_country",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "malware_name",
|
||||
output: "malware.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "matched_username",
|
||||
output: "security_finding.matched_username",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "md5",
|
||||
output: "file.md5",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "os",
|
||||
output: "os.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "os10",
|
||||
output: "device.os10",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "os11",
|
||||
output: "device.os11",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "policy",
|
||||
output: "policy.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "policy_id ",
|
||||
output: "policy.uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "protocol",
|
||||
output: "network_connection_info.protocol_name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "severity_level",
|
||||
output: "security_finding.severity",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "severity_level_id",
|
||||
output: "security_finding.severity_level_id",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "src_latitude",
|
||||
output: "geo_coordinates.src_latitude",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "src_longitude",
|
||||
output: "geo_coordinates.src_longitude",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "src_time",
|
||||
output: "security_finding.src_time",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "src_timezone",
|
||||
output: "location.src_timezone",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "src_zipcode",
|
||||
output: "location.src_zipcode",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "srcip",
|
||||
output: "src.ip.address",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "timestamp",
|
||||
output: "security_finding.time",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "transaction_id",
|
||||
output: "security_finding.transaction_id",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "true_obj_type",
|
||||
output: "file.type",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "url",
|
||||
output: "url.text",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "user",
|
||||
output: "user.name",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "user_id",
|
||||
output: "user.uid",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "userip",
|
||||
output: "user.ip",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "userkey",
|
||||
output: "user.key",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
{
|
||||
input: "local_sha256",
|
||||
output: "file.local_sha256",
|
||||
match: ".*",
|
||||
replace: "$0"
|
||||
},
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,39 +1,291 @@
|
||||
{
|
||||
attributes: {
|
||||
source: "okta"
|
||||
"dataSource.category": "security",
|
||||
"dataSource.name": "Okta",
|
||||
"attributes": {
|
||||
"dataSource.vendor": "Okta",
|
||||
"dataSource.name": "Okta System Log",
|
||||
"dataSource.category": "security",
|
||||
"metadata.product.vendor_name": "Okta",
|
||||
"metadata.product.name": "Okta System Log",
|
||||
"metadata.version": "1.0.0"
|
||||
},
|
||||
formats: [
|
||||
"formats": [
|
||||
{
|
||||
format: ".*${parse=dottedJson}{attrBlacklist=target}$"
|
||||
rewrites: [
|
||||
{ input: "actor.id", output: "user.account_uid", match: ".*", replace: "$0" },
|
||||
{ input: "actor.type", output: "user.account_type", match: ".*", replace: "$0" },
|
||||
{ input: "actor.alternateId", output: "user.email_addr", match: ".*", replace: "$0" },
|
||||
{ input: "actor.displayName", output: "user.name", match: ".*", replace: "$0" },
|
||||
{ input: "authenticationContext.authenticationStep", output: "authenticationStep", match: ".*", replace: "$0" },
|
||||
{ input: "authenticationContext.externalSessionId", output: "externalSessionId", match: ".*", replace: "$0" },
|
||||
{ input: "client.ipAddress", output: "client.ip", match: ".*", replace: "$0" },
|
||||
{ input: "client.userAgent.browser", output: "client.browser", match: ".*", replace: "$0" },
|
||||
{ input: "client.userAgent.os", output: "client.os", match: ".*", replace: "$0" },
|
||||
{ input: "client.userAgent.rawUserAgent", output: "client.userAgent", match: ".*", replace: "$0" },
|
||||
{ input: "client.zone", output: "client.location.zone", match: ".*", replace: "$0" },
|
||||
{ input: "client.geographicalContext.city", output: "client.location.city", match: ".*", replace: "$0" },
|
||||
{ input: "client.geographicalContext.country", output: "client.location.country", match: ".*", replace: "$0" },
|
||||
{ input: "client.geographicalContext.geolocation.lat", output: "client.location.lat", match: ".*", replace: "$0" },
|
||||
{ input: "client.geographicalContext.geolocation.lon", output: "client.location.lon", match: ".*", replace: "$0" },
|
||||
{ input: "client.geographicalContext.postalCode", output: "client.location.postal_code", match: ".*", replace: "$0" },
|
||||
{ input: "client.geographicalContext.state", output: "client.location.state", match: ".*", replace: "$0" },
|
||||
{ input: "displayMessage", output: "msg", match: ".*", replace: "$0" },
|
||||
{ input: "eventType", output: "category_name", match: ".*", replace: "$0" },
|
||||
{ input: "outcome.result", output: "result", match: ".*", replace: "$0" },
|
||||
{ input: "published", output: "time", match: ".*", replace: "$0" },
|
||||
{ input: "transaction.id", output: "type_uid", match: ".*", replace: "$0" },
|
||||
{ input: "transaction.type", output: "type_name", match: ".*", replace: "$0" },
|
||||
{ input: "uuid", output: "activity_id", match: ".*", replace: "$0" }
|
||||
"format": "$unmapped.{parse=json}$",
|
||||
"rewrites": [
|
||||
{
|
||||
"input": "unmapped.published",
|
||||
"output": "timestamp",
|
||||
"match": ".*",
|
||||
"replace": "$0"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
],
|
||||
"mappings": {
|
||||
"version": 1,
|
||||
"mappings": [
|
||||
{
|
||||
"predicate": "true",
|
||||
"transformations": [
|
||||
{
|
||||
"constant": {
|
||||
"value": 3002,
|
||||
"field": "class_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "Authentication",
|
||||
"field": "class_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 3,
|
||||
"field": "category_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": "Identity & Access Management",
|
||||
"field": "category_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.published",
|
||||
"to": "time"
|
||||
}
|
||||
},
|
||||
{
|
||||
"cast": {
|
||||
"field": "time",
|
||||
"type": "iso8601TimestampToEpochSec"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.uuid",
|
||||
"to": "metadata.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.eventType",
|
||||
"to": "type_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.displayMessage",
|
||||
"to": "message"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.actor.alternateId",
|
||||
"to": "user.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.actor.displayName",
|
||||
"to": "user.full_name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.actor.id",
|
||||
"to": "user.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.ipAddress",
|
||||
"to": "src_endpoint.ip"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.geographicalContext.city",
|
||||
"to": "src_endpoint.location.city"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.geographicalContext.state",
|
||||
"to": "src_endpoint.location.region"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.geographicalContext.country",
|
||||
"to": "src_endpoint.location.country"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.geographicalContext.postalCode",
|
||||
"to": "src_endpoint.location.postal_code"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.geographicalContext.geolocation.lat",
|
||||
"to": "src_endpoint.location.coordinates[0]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.geographicalContext.geolocation.lon",
|
||||
"to": "src_endpoint.location.coordinates[1]"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.userAgent.rawUserAgent",
|
||||
"to": "http_request.user_agent"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.userAgent.os.family",
|
||||
"to": "src_endpoint.os.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.client.userAgent.browser.family",
|
||||
"to": "http_request.user_agent"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.outcome.result",
|
||||
"to": "status"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.outcome.reason",
|
||||
"to": "status_detail"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.transaction.type",
|
||||
"to": "auth_protocol"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.transaction.id",
|
||||
"to": "session.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.authenticationContext.externalSessionId",
|
||||
"to": "session.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.debugContext.debugData.requestId",
|
||||
"to": "metadata.correlation_uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.debugContext.debugData.requestUri",
|
||||
"to": "http_request.url.path"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.target[0].displayName",
|
||||
"to": "dst_endpoint.name"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.target[0].alternateId",
|
||||
"to": "dst_endpoint.uid"
|
||||
}
|
||||
},
|
||||
{
|
||||
"copy": {
|
||||
"from": "unmapped.severity",
|
||||
"to": "severity"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "activity_id",
|
||||
"predicate": "unmapped.outcome.result = 'SUCCESS'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 2,
|
||||
"field": "activity_id",
|
||||
"predicate": "unmapped.outcome.result = 'FAILURE'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "severity_id",
|
||||
"predicate": "unmapped.outcome.result = 'SUCCESS'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 3,
|
||||
"field": "severity_id",
|
||||
"predicate": "unmapped.outcome.result = 'FAILURE'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 1,
|
||||
"field": "status_id",
|
||||
"predicate": "unmapped.outcome.result = 'SUCCESS'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"constant": {
|
||||
"value": 2,
|
||||
"field": "status_id",
|
||||
"predicate": "unmapped.outcome.result = 'FAILURE'"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"observables": {
|
||||
"fields": [
|
||||
{
|
||||
"name": "user.name",
|
||||
"type": "User"
|
||||
},
|
||||
{
|
||||
"name": "src_endpoint.ip",
|
||||
"type": "IP Address"
|
||||
},
|
||||
{
|
||||
"name": "http_request.user_agent",
|
||||
"type": "Other"
|
||||
},
|
||||
{
|
||||
"name": "session.uid",
|
||||
"type": "Other"
|
||||
},
|
||||
{
|
||||
"name": "dst_endpoint.name",
|
||||
"type": "Other"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
+86
-8
@@ -1,13 +1,91 @@
|
||||
{
|
||||
attributes: {
|
||||
"dataSource.vendor": "Palo Alto Networks",
|
||||
"dataSource.name": "Palo Alto Networks Prisma SASE",
|
||||
"dataSource.category": "security",
|
||||
// specify a time zone if the timestamps in your log are not in GMT
|
||||
//timezone: "Europe/Prague"
|
||||
|
||||
attributes: { "dataset.technology":"firewall", "dataset.vendor":"palo_alto", "dataset.app":"palo_alto" }
|
||||
|
||||
patterns: {
|
||||
//maps to high_resolution_timestamp:
|
||||
// timestamp: "\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}(\\+|-)\\d{2}:\\d{2}"
|
||||
tsPatternPA: "[A-za-z]+\\s+\\d{1,2} [\\d:]+"
|
||||
//application_characteristic can be a single value, a comma delimited list in quotes, or blank. Null value is handled by format: traffic-2, not by this pattern.
|
||||
application_characteristic: "(\".*\")|[^,]+"
|
||||
//description field from system log is wrapped in quotes and may contain commas"
|
||||
description: "(\".*\")"
|
||||
//discard future_use fields
|
||||
misc: "[^,]*"
|
||||
|
||||
}
|
||||
|
||||
formats: [
|
||||
//change pattern depending on the timestamp fomat
|
||||
{
|
||||
format: "$network_traffic.log_header$,$network_traffic.log_source_uid$,$event.type$,$network_traffic.sub_type_value$,$network_traffic.config_version_value$,$network_activity.time$,$src.ip.address$,$dst.ip.address$,$nat_src.device_ip$,$nat_dst.device_ip$,$rule.name$,$user.src_name$,$user.dst_name$,$network_traffic.app_name$,$network_traffic.virtual_system_location$,$network_traffic.from_zone$,$network_traffic.to_zone$,$network_traffic.inbound_if_value$,$network_traffic.outbound_if_value$,$network_traffic.log_set$,$session.uid$,$network_traffic.repeat_count$,$network_endpoint.src_port$,$network_endpoint.dst_port$,$nat_src.port$,$nat_dst.port$,$network_traffic.flags$,$network_connection_info.protocol_name$,$network_traffic.bytes$,$network_traffic.bytes_out$,$network_traffic.bytes_in$,$network_traffic.packets$,$session.created_time$,$network_traffic.total_time_elapsed$,$url.categories$,$metadata.sequence$,$network_traffic.action_flags$,$location.src_region$,$location.dst_region$,$network_traffic.packets_out$,$network_traffic.packets_in$,$network_traffic.session_end_reason_value$,$network_traffic.dg_hier_level_1$,$network_traffic.dg_hier_level_2$,$network_traffic.dg_hier_level_3$,$network_traffic.dg_hier_level_4$,$network_traffic.virtual_system_name$,$endpoint.name$,$network_traffic.action_source_value$,$source.uuid$,$destination.uuid$,$network_traffic.tunnel_id_imsi$,$network_traffic.monitor_tag_imei$,$session.parent_id$,$session.parent_start_time$,$network_traffic.tunnel_value$,$network_traffic.ep_association_uid$,$network_traffic.chunks$,$network_traffic.chunks_out$,$network_traffic.chunks_in$,$rule.uid$,$network_traffic.http2_connection$,$network_traffic.link_change_count$,$policy.uid$,$network_traffic.link_switches$,$network_traffic.sdwan_cluster$,$network_traffic.sdwan_device_type$,$network_traffic.sdwan_cluster_type$,$network_traffic.sdwan_site$,$network_traffic.dynusergroup_name$,$http_request.x_forwarded_for$,$source_device.category_name$,$source_device.profile$,$source_device.model$,$source_device.vendor_name$,$source_device.os_name$,$source_device.os_version$,$source_device.hostname$,$source_device.mac$,$destination_device.category_name$,$destination_device.profile$,$destination_device.model$,$destination_device.vendor_name$,$destination_device.os_name$,$destination_device.os_version$,$destination_device.hostname$,$destination_device.mac$,$container.uid$,$network_traffic.pod_namespace$,$network_traffic.pod_name$,$network_traffic.source_edl$,$network_traffic.destination_edl$,$host.uid$,$endpoint.serial_number$,$network_traffic.source_dynamic_address_group$,$network_traffic.destination_dynamic_address_group$,$network_traffic.ha_session_owner$,$network_traffic.timestamp_generated_high_res$,$network_traffic.nssai_network_slice_type_value$,$network_traffic.nssai_network_slice_differentiator_value$",
|
||||
halt: true,
|
||||
}
|
||||
format: ".*$timestamp=tsPatternPA$(\\,)*",
|
||||
},
|
||||
{
|
||||
//match all fields. application_characteristic can be a single value, or a comma delimited list in quotes.
|
||||
id: "traffic-1",
|
||||
attributes: { type: "TRAFFIC", format: "traffic-1" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$firewall_serial_number$,TRAFFIC,$sub_type$,\\d+,$generate_time$,$source_address$,$destination_address$,$source_nat_address$,$destination_nat_address$,$rule_name$,$source_user$,$destination_user$,$application$,$virtual_system$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$log_action$,$time_stamp$,$session_id$,$repeat_count$,$source_port$,$destination_port$,$source_nat_port$,$destination_nat_port$,$flag$,$protocol$,$action$,$bytes$,$bytes_sent$,$bytes_received$,$packets$,$start_time$,$elapsed_time$,$category$,$test$,$sequence_number$,$action_flags$,$source_country$,$destination_country$,,$packets_sent$,$packets_received$,$session_end_reason$,\\d+,\\d+,\\d+,\\d+,$virtual_system_name$,$device_name$,$action_source$,$source_vm_uuid$,$destination_vm_uuid$,$tunnel_id_imsi$,$monitor_tag_imei$,$parent_session_id$,$parent_start_time$,$tunnel_type$,$sctp_association_id$,$sctp_chunks$,$sctp_chunks_sent$,$sctp_chunks_received$,$rule_uuid$,$http_2_connection$,$app_flap_count$,$policy_id$,$link_switches$,$sd-wan_cluster$,$sd-wan_device_type$,$sd-wan_cluster_type$,$sd-wan_site$,$dynamic_user_group_name$,$xff_address$,$source_device_category$,$source_device_profile$,$source_device_model$,$source_device_vendor$,$source_device_os_family$,$source_device_os_version$,$source_hostname$,$source_mac_address$,$destination_device_category$,$destination_device_profile$,$destination_device_model$,$destination_device_vendor$,$destination_device_os_family$,$destination_device_os_version$,$destination_hostname$,$destination_mac_address$,$container_id$,$pod_namespace$,$pod_name$,$source_external_dynamic_list$,$destination_external_dynamic_list$,$host_id$,$serial_number$,$source_dynamic_address_group$,$destination_dynamic_address_group$,$session_owner$,$high_resolution_timestamp$,$a_slice_service_type$,$a_slice_differentiator$,$application_subcategory$,$application_category$,$application_technology$,$application_risk$,$application_characteristic=application_characteristic$,$application_container$,$tunneled_application$,$application_saas$,$application_sanctioned_state$,$offloaded$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
//dont match on application_characteristic for cases where is it blank.
|
||||
id: "traffic-2",
|
||||
attributes: { type: "TRAFFIC", format: "traffic-2" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$firewall_serial_number$,TRAFFIC,$sub_type$,\\d+,$generate_time$,$source_address$,$destination_address$,$source_nat_address$,$destination_nat_address$,$rule_name$,$source_user$,$destination_user$,$application$,$virtual_system$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$log_action$,$time_stamp$,$session_id$,$repeat_count$,$source_port$,$destination_port$,$source_nat_port$,$destination_nat_port$,$flag$,$protocol$,$action$,$bytes$,$bytes_sent$,$bytes_received$,$packets$,$start_time$,$elapsed_time$,$category$,$test$,$sequence_number$,$action_flags$,$source_country$,$destination_country$,,$packets_sent$,$packets_received$,$session_end_reason$,\\d+,\\d+,\\d+,\\d+,$virtual_system_name$,$device_name$,$action_source$,$source_vm_uuid$,$destination_vm_uuid$,$tunnel_id_imsi$,$monitor_tag_imei$,$parent_session_id$,$parent_start_time$,$tunnel_type$,$sctp_association_id$,$sctp_chunks$,$sctp_chunks_sent$,$sctp_chunks_received$,$rule_uuid$,$http_2_connection$,$app_flap_count$,$policy_id$,$link_switches$,$sd-wan_cluster$,$sd-wan_device_type$,$sd-wan_cluster_type$,$sd-wan_site$,$dynamic_user_group_name$,$xff_address$,$source_device_category$,$source_device_profile$,$source_device_model$,$source_device_vendor$,$source_device_os_family$,$source_device_os_version$,$source_hostname$,$source_mac_address$,$destination_device_category$,$destination_device_profile$,$destination_device_model$,$destination_device_vendor$,$destination_device_os_family$,$destination_device_os_version$,$destination_hostname$,$destination_mac_address$,$container_id$,$pod_namespace$,$pod_name$,$source_external_dynamic_list$,$destination_external_dynamic_list$,$host_id$,$serial_number$,$source_dynamic_address_group$,$destination_dynamic_address_group$,$session_owner$,$high_resolution_timestamp$,$a_slice_service_type$,$a_slice_differentiator$,$application_subcategory$,$application_category$,$application_technology$,$application_risk$,,$application_container$,$tunneled_application$,$application_saas$,$application_sanctioned_state$,$offloaded$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
id: "system",
|
||||
attributes: { type: "SYSTEM", format: "system" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$serial_number$,SYSTEM,$content_threat_type$,.*,$generated_time$,$virtual_system$,$event_id$,$object$,.*,.*,$module$,$severity$,$description=description$,$sequence_number$,$action_flags$,$device_group_hierarchy_level_1$,$device_group_hierarchy_level_2$,$device_group_hierarchy_level_3$,$device_group_hierarchy_level_4$,$virtual_system_name$,$device_name$,.*,.*,$high_resolution_timestamp$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
//matches THREAT logs with comma surround lists in application_characteristic and url_category_list.
|
||||
// Matches THREAT logs with commas surrounding user_agent
|
||||
//PAN OS 10.2 will add $cloud_report_id to the end of this log format. If this no longer matches threat logs, check for the extra field at the end.
|
||||
id: "threat-0",
|
||||
attributes: { type: "THREAT", format: "threat-0" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$serial_number$,THREAT,$threat_content_type$,\\d+,$Generated_time$,$source_address$,$destination_address$,$nat_source_address$,$nat_destination_address$,$rule_name$,$source_user$,$destination_user$,$application$,$virtual_system$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$log_action$,$misc=misc$,$Session_id$,$repeat_count$,$source_port$,$destination_port$,$nat_source_port$,$nat_destination_port$,$flags$,$ip_protocol$,$action$,$url_filename$,$threat_id$,$category$,$severity$,$direction$,$sequence_number$,$action_flags$,$source_location$,$destination_location$,$misc=misc$,$Content_type$,$pcap_id$,$file_digest$,$cloud$,$url_index$,\"$user_agent$\",$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report_id$,$device_group_hierarchy_level_1$,$device_group_hierarchy_level_2$,$device_group_hierarchy_level_3$,$device_group_hierarchy_level_4$,$virtual_system_name$,$device_name$,$misc=misc$,$Source_vm_uuid$,$destination_vm_uuid$,$http_method$,$tunnel_id_imsi$,$monitor_tag_imei$,$parent_session_id$,$parent_start_time$,$tunnel_type$,$threat_category$,$content_version$,$misc=misc$,$Sctp_association_id$,$payload_protocol_id$,$http_headers$,\"$url_category_list$\",$rule_uuid$,$http_2_connection$,$dynamic_user_group_name$,$xff_address$,$source_device_category$,$source_device_profile$,$source_device_model$,$source_device_vendor$,$source_device_os_family$,$source_device_os_version$,$source_hostname$,$source_mac_address$,$destination_device_category$,$destination_device_profile$,$destination_device_model$,$destination_device_vendor$,$destination_device_os_family$,$destination_device_os_version$,$destination_hostname$,$destination_mac_address$,$container_id$,$pod_namespace$,$pod_name$,$source_external_dynamic_list$,$destination_external_dynamic_list$,$host_id$,$serial_number$,$domain_edl$,$source_dynamic_address_group$,$destination_dynamic_address_group$,$partial_hash$,$high_resolution_timestamp$,$reason$,$justification$,$a_slice_service_type$,$application_subcategory$,$application_category$,$application_technology$,$application_risk$,\"$application_characteristic$\",$application_container$,$tunneled_application$,$application_saas$,$application_sanctioned_state$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
//matches THREAT logs with comma surround lists in application_characteristic and url_category_list.
|
||||
//PAN OS 10.2 will add $cloud_report_id to the end of this log format. If this no longer matches threat logs, check for the extra field at the end.
|
||||
id: "threat-1",
|
||||
attributes: { type: "THREAT", format: "threat-1" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$serial_number$,THREAT,$threat_content_type$,\\d+,$Generated_time$,$source_address$,$destination_address$,$nat_source_address$,$nat_destination_address$,$rule_name$,$source_user$,$destination_user$,$application$,$virtual_system$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$log_action$,$misc=misc$,$Session_id$,$repeat_count$,$source_port$,$destination_port$,$nat_source_port$,$nat_destination_port$,$flags$,$ip_protocol$,$action$,$url_filename$,$threat_id$,$category$,$severity$,$direction$,$sequence_number$,$action_flags$,$source_location$,$destination_location$,$misc=misc$,$Content_type$,$pcap_id$,$file_digest$,$cloud$,$url_index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report_id$,$device_group_hierarchy_level_1$,$device_group_hierarchy_level_2$,$device_group_hierarchy_level_3$,$device_group_hierarchy_level_4$,$virtual_system_name$,$device_name$,$misc=misc$,$Source_vm_uuid$,$destination_vm_uuid$,$http_method$,$tunnel_id_imsi$,$monitor_tag_imei$,$parent_session_id$,$parent_start_time$,$tunnel_type$,$threat_category$,$content_version$,$misc=misc$,$Sctp_association_id$,$payload_protocol_id$,$http_headers$,\"$url_category_list$\",$rule_uuid$,$http_2_connection$,$dynamic_user_group_name$,$xff_address$,$source_device_category$,$source_device_profile$,$source_device_model$,$source_device_vendor$,$source_device_os_family$,$source_device_os_version$,$source_hostname$,$source_mac_address$,$destination_device_category$,$destination_device_profile$,$destination_device_model$,$destination_device_vendor$,$destination_device_os_family$,$destination_device_os_version$,$destination_hostname$,$destination_mac_address$,$container_id$,$pod_namespace$,$pod_name$,$source_external_dynamic_list$,$destination_external_dynamic_list$,$host_id$,$serial_number$,$domain_edl$,$source_dynamic_address_group$,$destination_dynamic_address_group$,$partial_hash$,$high_resolution_timestamp$,$reason$,$justification$,$a_slice_service_type$,$application_subcategory$,$application_category$,$application_technology$,$application_risk$,\"$application_characteristic$\",$application_container$,$tunneled_application$,$application_saas$,$application_sanctioned_state$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
//matches THREAT logs without comma surround list in url_category_list.
|
||||
//PAN OS 10.2 will add $cloud_report_id to the end of this log format. If this no longer matches threat logs, check for the extra field at the end.
|
||||
id: "threat-2",
|
||||
attributes: { type: "THREAT", format: "threat-2" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$serial_number$,THREAT,$threat_content_type$,\\d+,$Generated_time$,$source_address$,$destination_address$,$nat_source_address$,$nat_destination_address$,$rule_name$,$source_user$,$destination_user$,$application$,$virtual_system$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$log_action$,$misc=misc$,$Session_id$,$repeat_count$,$source_port$,$destination_port$,$nat_source_port$,$nat_destination_port$,$flags$,$ip_protocol$,$action$,$url_filename$,$threat_id$,$category$,$severity$,$direction$,$sequence_number$,$action_flags$,$source_location$,$destination_location$,$misc=misc$,$Content_type$,$pcap_id$,$file_digest$,$cloud$,$url_index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report_id$,$device_group_hierarchy_level_1$,$device_group_hierarchy_level_2$,$device_group_hierarchy_level_3$,$device_group_hierarchy_level_4$,$virtual_system_name$,$device_name$,$misc=misc$,$Source_vm_uuid$,$destination_vm_uuid$,$http_method$,$tunnel_id_imsi$,$monitor_tag_imei$,$parent_session_id$,$parent_start_time$,$tunnel_type$,$threat_category$,$content_version$,$misc=misc$,$Sctp_association_id$,$payload_protocol_id$,$http_headers$,$url_category_list$,$rule_uuid$,$http_2_connection$,$dynamic_user_group_name$,$xff_address$,$source_device_category$,$source_device_profile$,$source_device_model$,$source_device_vendor$,$source_device_os_family$,$source_device_os_version$,$source_hostname$,$source_mac_address$,$destination_device_category$,$destination_device_profile$,$destination_device_model$,$destination_device_vendor$,$destination_device_os_family$,$destination_device_os_version$,$destination_hostname$,$destination_mac_address$,$container_id$,$pod_namespace$,$pod_name$,$source_external_dynamic_list$,$destination_external_dynamic_list$,$host_id$,$serial_number$,$domain_edl$,$source_dynamic_address_group$,$destination_dynamic_address_group$,$partial_hash$,$high_resolution_timestamp$,$reason$,$justification$,$a_slice_service_type$,$application_subcategory$,$application_category$,$application_technology$,$application_risk$,\"$application_characteristic$\",$application_container$,$tunneled_application$,$application_saas$,$application_sanctioned_state$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
//matches THREAT logs without comma surround list in url_category_list or application_characteristic.
|
||||
//PAN OS 10.2 will add $cloud_report_id to the end of this log format. If this no longer matches threat logs, check for the extra field at the end.
|
||||
id: "threat-3",
|
||||
attributes: { type: "THREAT", format: "threat-3" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$serial_number$,THREAT,$threat_content_type$,\\d+,$Generated_time$,$source_address$,$destination_address$,$nat_source_address$,$nat_destination_address$,$rule_name$,$source_user$,$destination_user$,$application$,$virtual_system$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$log_action$,$misc=misc$,$Session_id$,$repeat_count$,$source_port$,$destination_port$,$nat_source_port$,$nat_destination_port$,$flags$,$ip_protocol$,$action$,$url_filename$,$threat_id$,$category$,$severity$,$direction$,$sequence_number$,$action_flags$,$source_location$,$destination_location$,$misc=misc$,$Content_type$,$pcap_id$,$file_digest$,$cloud$,$url_index$,$user_agent$,$file_type$,$x-forwarded-for$,$referer$,$sender$,$subject$,$recipient$,$report_id$,$device_group_hierarchy_level_1$,$device_group_hierarchy_level_2$,$device_group_hierarchy_level_3$,$device_group_hierarchy_level_4$,$virtual_system_name$,$device_name$,$misc=misc$,$Source_vm_uuid$,$destination_vm_uuid$,$http_method$,$tunnel_id_imsi$,$monitor_tag_imei$,$parent_session_id$,$parent_start_time$,$tunnel_type$,$threat_category$,$content_version$,$misc=misc$,$Sctp_association_id$,$payload_protocol_id$,$http_headers$,$url_category_list$,$rule_uuid$,$http_2_connection$,$dynamic_user_group_name$,$xff_address$,$source_device_category$,$source_device_profile$,$source_device_model$,$source_device_vendor$,$source_device_os_family$,$source_device_os_version$,$source_hostname$,$source_mac_address$,$destination_device_category$,$destination_device_profile$,$destination_device_model$,$destination_device_vendor$,$destination_device_os_family$,$destination_device_os_version$,$destination_hostname$,$destination_mac_address$,$container_id$,$pod_namespace$,$pod_name$,$source_external_dynamic_list$,$destination_external_dynamic_list$,$host_id$,$serial_number$,$domain_edl$,$source_dynamic_address_group$,$destination_dynamic_address_group$,$partial_hash$,$high_resolution_timestamp$,$reason$,$justification$,$a_slice_service_type$,$application_subcategory$,$application_category$,$application_technology$,$application_risk$,$application_characteristic$,$application_container$,$tunneled_application$,$application_saas$,$application_sanctioned_state$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
id: "userid",
|
||||
attributes: { type: "USERID", format: "userid" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$serial_number$,USERID,$threat_content_type$,$misc=misc$,$generated_time$,$virtual_system$,$source_ip$,$user$,$data_source_name$,$event_id$,$repeat_count$,$time_out_threshold$,$source_port$,$destination_port$,$data_source$,$data_source_type$,$sequence_number$,$action_flags$,$device_group_hierarchy_level_1$,$device_group_hierarchy_level_2$,$device_group_hierarchy_level_3$,$device_group_hierarchy_level_4$,$virtual_system_name$,$device_name$,$virtual_system_id$,$factor_type$,$factor_completion_time$,$factor_number$,$user_group_flags$,$user_by_source$,$misc=misc$,$high_resolution_timestamp$",
|
||||
halt: true
|
||||
},
|
||||
{
|
||||
//dont match on application_characteristic for cases where is it blank.
|
||||
id: "basic",
|
||||
attributes: { format: "basic" },
|
||||
format: "\\<[^\\>]+\\>$timestamp=tsPatternPA$ $hostname$ $log_version$,$receive_time$,$firewall_serial_number$,$type$,$sub_type$,\\d+,$generate_time$,$source_address$,$destination_address$,$source_nat_address$,$destination_nat_address$,$rule_name$,$source_user$,$destination_user$,$application$,$virtual_system$,$source_zone$,$destination_zone$,$inbound_interface$,$outbound_interface$,$log_action$,$time_stamp$,$session_id$,$repeat_count$,$source_port$,$destination_port$,$source_nat_port$,$destination_nat_port$,$flag$,$protocol$,$action$,.*",
|
||||
},
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user