mirror of
https://github.com/marcredhat/SIEM-toolkit-patched
synced 2026-06-08 20:37:12 +00:00
marc
7d19c57a5d
Adds an asyncio background task that re-runs the heavy Ingest Dashboard
queries every ~4 min (just under the 5 min TTL) so the in-process cache
is always populated. First user hit on any dashboard widget then returns
from cache (single-digit ms) instead of waiting 30-60s for SDL.
Components:
- backend/services/prewarmer.py: standalone module, opt-in via
INGEST_PREWARM=1; configurable windows via INGEST_PREWARM_HOURS /
INGEST_PREWARM_DAYS / INGEST_PREWARM_DAILY_VOLUME_DAYS and interval
via INGEST_PREWARM_INTERVAL_SECONDS. Logs through the uvicorn logger
so cycles are visible in 'docker logs'.
- backend/main.py: spawn the task on FastAPI startup.
- docker-compose.yml: forward INGEST_PREWARM* env vars to the
backend service (default off).
Observed on a busy tenant with INGEST_PREWARM=1, default windows:
top-sources?days=7 first hit after restart: ~39s -> ~8ms (cache warm).
Defaults to disabled (INGEST_PREWARM=0) so existing users see no
behaviour change.
54 lines
1.5 KiB
YAML
54 lines
1.5 KiB
YAML
services:
|
|
frontend:
|
|
build: ./frontend
|
|
ports:
|
|
- "3001:3000"
|
|
depends_on:
|
|
- backend
|
|
|
|
backend:
|
|
build: ./backend
|
|
ports:
|
|
- "8001:8000"
|
|
environment:
|
|
- S1_API_TOKEN=${S1_API_TOKEN}
|
|
- S1_BASE_URL=${S1_BASE_URL}
|
|
- SDL_XDR_URL=${SDL_XDR_URL}
|
|
- SDL_LOG_READ_KEY=${SDL_LOG_READ_KEY}
|
|
- SDL_CONFIG_READ_KEY=${SDL_CONFIG_READ_KEY}
|
|
- SDL_PQ_TIMEOUT=${SDL_PQ_TIMEOUT:-600}
|
|
- SDL_PQ_TIMEOUT_RETRIES=${SDL_PQ_TIMEOUT_RETRIES:-1}
|
|
- ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}
|
|
- DATABASE_URL=postgresql://siem:siem@db:5432/siem
|
|
- DETECTIONS_FILE=/app/data/detections.json
|
|
- INGEST_PREWARM=${INGEST_PREWARM:-0}
|
|
- INGEST_PREWARM_HOURS=${INGEST_PREWARM_HOURS:-1,24,168}
|
|
- INGEST_PREWARM_DAYS=${INGEST_PREWARM_DAYS:-7}
|
|
- INGEST_PREWARM_DAILY_VOLUME_DAYS=${INGEST_PREWARM_DAILY_VOLUME_DAYS:-5}
|
|
- INGEST_PREWARM_INTERVAL_SECONDS=${INGEST_PREWARM_INTERVAL_SECONDS:-240}
|
|
depends_on:
|
|
db:
|
|
condition: service_healthy
|
|
volumes:
|
|
- ./parsers:/app/parsers
|
|
- ./.env:/app/.env
|
|
- ./data:/app/data:ro
|
|
|
|
db:
|
|
image: postgres:16-alpine
|
|
environment:
|
|
- POSTGRES_DB=siem
|
|
- POSTGRES_USER=siem
|
|
- POSTGRES_PASSWORD=siem
|
|
volumes:
|
|
- pgdata:/var/lib/postgresql/data
|
|
- ./db/init.sql:/docker-entrypoint-initdb.d/init.sql
|
|
healthcheck:
|
|
test: ["CMD-SHELL", "pg_isready -U siem"]
|
|
interval: 5s
|
|
timeout: 5s
|
|
retries: 5
|
|
|
|
volumes:
|
|
pgdata:
|