"""
build_sources.py — NOX Framework · Production Source Builder
Generates individual JSON plugin files for every intelligence source.
"""
from __future__ import annotations
import json
import os
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, List, Literal, Optional
from pydantic import BaseModel, Field, field_validator, model_validator
# ── Shared placeholder constant ────────────────────────────────────────
# Import from config_handler so the string is defined in exactly one place.
# Fall back to the literal if the helper isn't on sys.path yet (e.g. bare
# invocation before sources/ exists).
try:
sys.path.insert(0, str(Path(__file__).parent))
from sources.helpers.config_handler import (
UNIVERSAL_PLACEHOLDER,
_APIKEYS_FILE,
_default_store,
_write_store,
)
except ImportError:
UNIVERSAL_PLACEHOLDER = "INSERT_API_KEY_HERE"
_APIKEYS_FILE = None
_default_store = None
_write_store = None
# ---------------------------------------------------------------------------
# Pydantic Schema
# ---------------------------------------------------------------------------
HttpMethod = Literal["GET", "POST", "PUT", "DELETE"]
InputType = Literal["email", "ip", "domain", "hash", "username", "phone", "url", "cve", "any"]
ReliabilityScore = Literal[1, 2, 3, 4, 5]
class SourceConfig(BaseModel):
# ── Mandatory core ──────────────────────────────────────────────────────
name: str
category: str
endpoint: str
method: HttpMethod
requires_auth: bool
selectors: Dict[str, str]
# ── Request plumbing ────────────────────────────────────────────────────
rate_limit: float = 1.0
headers: Dict[str, str] = Field(default_factory=dict)
payload_template: Optional[Dict[str, Any]] = None
raw_payload: Optional[str] = None
regex_pattern: Optional[str] = None
api_key_slots: List[str] = Field(default_factory=list)
# ── Typing & pivoting ───────────────────────────────────────────────────
input_type: InputType = "any"
output_type: List[str] = Field(default_factory=list)
normalization_map: Dict[str, str] = Field(default_factory=dict)
tags: List[str] = Field(default_factory=list)
# ── Resilience ──────────────────────────────────────────────────────────
health_check_url: str
expected_status: int = 200
reliability_score: ReliabilityScore = 5
is_volatile: Optional[bool] = None # omitted when False
bypass_required: Optional[List[str]] = None # omitted when empty
user_agent_type: Optional[str] = None # omitted when absent
backup_endpoints: List[str] = Field(default_factory=list)
confidence: Optional[float] = None
query_transform: Optional[str] = None # e.g. "md5_lower"
# Two-phase poll support (e.g. IntelX: POST → job_id → GET results)
poll_endpoint: Optional[str] = None
poll_id_field: Optional[str] = None
poll_id_param: Optional[str] = None
poll_json_root: Optional[str] = None
@field_validator("reliability_score")
@classmethod
def _score_range(cls, v: int) -> int:
if not 1 <= v <= 5:
raise ValueError("reliability_score must be 1–5")
return v
@model_validator(mode="after")
def _validate_source(self) -> "SourceConfig":
if self.method.upper() == "GET" and "{target}" not in self.endpoint:
raise ValueError(
f"'{self.name}': GET endpoint must contain {{target}} placeholder: {self.endpoint!r}"
)
if self.is_volatile and self.reliability_score > 4:
raise ValueError(
f"'{self.name}': is_volatile sources must have reliability_score ≤ 4"
)
return self
def to_json(self) -> str:
data = self.model_dump(exclude_none=True)
# Drop falsy optional fields
for key in ("is_volatile", "bypass_required", "user_agent_type", "query_transform"):
if not data.get(key):
data.pop(key, None)
data["confidence"] = (
round(self.confidence, 2)
if self.confidence is not None
else round(0.4 + (self.reliability_score - 1) * 0.15, 2)
)
return json.dumps(data, indent=4)
# ---------------------------------------------------------------------------
# Builder helpers (_base → requires_auth=False, _auth → requires_auth=True)
# ---------------------------------------------------------------------------
def _mk(
name: str, category: str, endpoint: str, method: HttpMethod,
selectors: Dict[str, str], *,
requires_auth: bool,
rate_limit: float = 1.0,
headers: Optional[Dict[str, str]] = None,
payload_template: Optional[Dict[str, Any]] = None,
raw_payload: Optional[str] = None,
regex_pattern: Optional[str] = None,
api_key_slots: Optional[List[str]] = None,
input_type: InputType = "any",
output_type: Optional[List[str]] = None,
normalization_map: Optional[Dict[str, str]] = None,
tags: Optional[List[str]] = None,
health_check_url: Optional[str] = None,
expected_status: int = 200,
reliability_score: ReliabilityScore = 5,
is_volatile: bool = False,
bypass_required: Optional[List[str]] = None,
user_agent_type: Optional[str] = None,
backup_endpoints: Optional[List[str]] = None,
poll_endpoint: Optional[str] = None,
poll_id_field: Optional[str] = None,
poll_id_param: Optional[str] = None,
poll_json_root: Optional[str] = None,
confidence: Optional[float] = None,
query_transform: Optional[str] = None,
) -> SourceConfig:
return SourceConfig(
name=name, category=category, endpoint=endpoint, method=method,
requires_auth=requires_auth, selectors=selectors,
rate_limit=rate_limit,
headers=headers or {},
payload_template=payload_template,
raw_payload=raw_payload,
regex_pattern=regex_pattern,
api_key_slots=api_key_slots or [],
input_type=input_type,
output_type=output_type or [],
normalization_map=normalization_map or {},
tags=tags or [],
health_check_url=health_check_url or endpoint.split("{")[0].rstrip("/?"),
expected_status=expected_status,
reliability_score=reliability_score,
is_volatile=is_volatile or None,
bypass_required=bypass_required or None,
user_agent_type=user_agent_type,
backup_endpoints=backup_endpoints or [],
poll_endpoint=poll_endpoint,
poll_id_field=poll_id_field,
poll_id_param=poll_id_param,
poll_json_root=poll_json_root,
confidence=confidence,
query_transform=query_transform or None,
)
def _base(name, category, endpoint, method, selectors, **kw) -> SourceConfig:
return _mk(name, category, endpoint, method, selectors, requires_auth=False, **kw)
def _auth(name, category, endpoint, method, selectors, **kw) -> SourceConfig:
return _mk(name, category, endpoint, method, selectors, requires_auth=True, **kw)
# ---------------------------------------------------------------------------
# FREE / PUBLIC SOURCES
# ---------------------------------------------------------------------------
FREE_PUBLIC_SOURCES: List[SourceConfig] = [
_base("crt_sh", "certificate_transparency",
"https://crt.sh/?q={target}&output=json", "GET",
{"domains": "$.*.name_value"},
headers={"Accept": "application/json"},
input_type="domain", output_type=["domain"],
normalization_map={"name_value": "domain"},
tags=["passive", "fast"],
health_check_url="https://crt.sh", reliability_score=3, is_volatile=True),
_base("hackertarget_hostsearch", "dns_recon",
"https://api.hackertarget.com/hostsearch/?q={target}", "GET",
{"hosts": "text_lines"},
input_type="domain", output_type=["ip", "domain"],
tags=["passive", "fast"],
health_check_url="https://api.hackertarget.com", reliability_score=4),
_base("hackertarget_reverseip", "dns_recon",
"https://api.hackertarget.com/reverseiplookup/?q={target}", "GET",
{"domains": "text_lines"},
input_type="ip", output_type=["domain"],
tags=["passive"],
health_check_url="https://api.hackertarget.com", reliability_score=4),
_base("hackertarget_dnslookup", "dns_recon",
"https://api.hackertarget.com/dnslookup/?q={target}", "GET",
{"records": "text_lines"},
input_type="domain", output_type=["ip"],
tags=["passive", "fast"],
health_check_url="https://api.hackertarget.com", reliability_score=4),
_base("hackertarget_whois", "whois",
"https://api.hackertarget.com/whois/?q={target}", "GET",
{"raw": "text_lines"},
input_type="domain", output_type=["email", "domain"],
tags=["passive"],
health_check_url="https://api.hackertarget.com", reliability_score=4),
_base("alienvault_otx_domain", "threat_intel",
"https://otx.alienvault.com/api/v1/indicators/domain/{target}/general", "GET",
{"pulses": "$.pulse_info.count", "tags": "$.tags"},
input_type="domain", output_type=["domain", "ip"],
tags=["passive", "threat"],
health_check_url="https://otx.alienvault.com", reliability_score=5),
_base("alienvault_otx_ip", "threat_intel",
"https://otx.alienvault.com/api/v1/indicators/IPv4/{target}/general", "GET",
{"asn": "$.asn", "country": "$.country_name"},
input_type="ip", output_type=["domain"],
tags=["passive", "threat"],
health_check_url="https://otx.alienvault.com", reliability_score=5),
_base("alienvault_otx_malware", "threat_intel",
"https://otx.alienvault.com/api/v1/indicators/file/{target}/analysis", "GET",
{"malware": "$.analysis.malware"},
input_type="hash", output_type=["hash"],
tags=["passive", "threat"],
health_check_url="https://otx.alienvault.com", reliability_score=5),
_base("alienvault_otx_user", "social",
"https://otx.alienvault.com/api/v1/users/{target}/general", "GET",
{"pulses": "$.pulse_count"},
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://otx.alienvault.com", reliability_score=5),
_base("urlscan_search", "url_analysis",
"https://urlscan.io/api/v1/search/?q={target}", "GET",
{"urls": "$.results[*].page.url"},
rate_limit=2.0,
input_type="domain", output_type=["url", "ip", "domain"],
tags=["passive"],
health_check_url="https://urlscan.io", reliability_score=5),
_base("pulsedive", "threat_intel",
"https://pulsedive.com/api/info.php?indicator={target}", "GET",
{"risk": "$.risk", "threats": "$.threats"},
rate_limit=2.0,
input_type="any", output_type=["domain", "ip"],
tags=["passive", "threat"],
health_check_url="https://pulsedive.com", reliability_score=4),
_base("xposedornot", "breach_data",
"https://api.xposedornot.com/v1/breach-analytics?email={target}", "GET",
{"breaches": "$.ExposedBreaches.breaches_details"},
rate_limit=2.0,
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"},
input_type="email", output_type=["email", "domain"],
normalization_map={"breach": "breach_name", "domain": "domain",
"xposed_date": "breach_date", "xposed_data": "data_types",
"password_risk": "password_risk"},
tags=["passive", "stealth"],
health_check_url="https://api.xposedornot.com", reliability_score=4, confidence=0.75),
_base("hudsonrock_osint", "breach_data",
"https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-email?email={target}", "GET",
{"stealers": "$.stealers"},
rate_limit=30.0,
headers={"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"},
input_type="email", output_type=["email", "domain", "username"],
normalization_map={"stealers": "breach_record"},
tags=["passive", "stealth"],
health_check_url="https://cavalier.hudsonrock.com", reliability_score=3, is_volatile=True),
_base("ipinfo_io", "geolocation",
"https://ipinfo.io/{target}/json", "GET",
{"org": "$.org", "city": "$.city"},
input_type="ip", output_type=["domain"],
normalization_map={"org": "asn_org", "city": "geo_city"},
tags=["passive", "fast"],
health_check_url="https://ipinfo.io", reliability_score=5),
_base("ipapi_co", "geolocation",
"https://ipapi.co/{target}/json/", "GET",
{"asn": "$.asn", "org": "$.org"},
headers={"User-Agent": "Mozilla/5.0"},
input_type="ip", output_type=["domain"],
normalization_map={"asn": "asn_number", "org": "asn_org"},
tags=["passive", "fast"],
health_check_url="https://ipapi.co", reliability_score=4),
_base("ripestat_ip", "network",
"https://stat.ripe.net/data/prefix-overview/data.json?resource={target}", "GET",
{"asns": "$.data.asns[*].asn", "holder": "$.data.asns[0].holder"},
input_type="ip", output_type=["ip"],
normalization_map={"asn": "asn_number", "holder": "asn_org"},
tags=["passive", "infrastructure"],
health_check_url="https://stat.ripe.net", reliability_score=5),
_auth("emailrep_io", "email_rep",
"https://emailrep.io/{target}", "GET",
{"reputation": "$.reputation"},
rate_limit=2.0,
headers={"Key": "{EMAILREP_API_KEY}"},
api_key_slots=["{EMAILREP_API_KEY}"],
input_type="email", output_type=["email"],
normalization_map={"reputation": "email_reputation"},
tags=["passive", "fast"],
health_check_url="https://emailrep.io", reliability_score=4),
_base("github_users", "social",
"https://api.github.com/users/{target}", "GET",
{"bio": "$.bio", "blog": "$.blog"},
rate_limit=2.0, headers={"User-Agent": "NOX"},
input_type="username", output_type=["username", "domain"],
tags=["passive", "fast"],
health_check_url="https://api.github.com", reliability_score=5),
_base("reddit_user", "social",
"https://www.reddit.com/user/{target}/about.json", "GET",
{"karma": "$.data.total_karma"},
rate_limit=2.0, headers={"User-Agent": "NOX"},
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://www.reddit.com", reliability_score=4),
_base("gravatar", "social",
"https://www.gravatar.com/{target}.json", "GET",
{"name": "$.entry[0].displayName"},
rate_limit=2.0,
input_type="email", output_type=["username"],
query_transform="md5_lower",
tags=["passive"],
health_check_url="https://www.gravatar.com", reliability_score=4),
_base("anubis_subdomains", "dns_recon",
"https://jldc.me/anubis/subdomains/{target}", "GET",
{"subdomains": "$.*"},
input_type="domain", output_type=["domain"],
tags=["passive"],
health_check_url="https://jldc.me", reliability_score=3, is_volatile=True),
_base("sublist3r_api", "dns_recon",
"https://api.sublist3r.com/search.php?domain={target}", "GET",
{"subdomains": "$.*"},
input_type="domain", output_type=["domain"],
tags=["passive"],
health_check_url="https://api.sublist3r.com", reliability_score=3, is_volatile=True),
_base("keybase_lookup", "social",
"https://keybase.io/_/api/1.0/user/lookup.json?username={target}", "GET",
{"id": "$.them[0].id"},
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://keybase.io", reliability_score=4),
_base("keybase_proofs", "social",
"https://keybase.io/_/api/1.0/user/lookup.json?usernames={target}", "GET",
{"proofs": "$.them[0].proofs_summary.all[*].namestr"},
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://keybase.io", reliability_score=4),
_base("maltiverse_ip", "threat_intel",
"https://api.maltiverse.com/ip/{target}", "GET",
{"classification": "$.classification"},
input_type="ip", output_type=["ip"],
tags=["passive", "threat"],
health_check_url="https://api.maltiverse.com", reliability_score=3),
_base("threatminer_domain", "threat_intel",
"https://api.threatminer.org/v2/domain.php?q={target}&rt=1", "GET",
{"ips": "$.results"},
input_type="domain", output_type=["ip"],
tags=["passive", "threat"],
health_check_url="https://api.threatminer.org", reliability_score=3, is_volatile=True),
_base("threatminer_ip", "threat_intel",
"https://api.threatminer.org/v2/host.php?q={target}&rt=1", "GET",
{"urls": "$.results"},
input_type="ip", output_type=["url"],
tags=["passive", "threat"],
health_check_url="https://api.threatminer.org", reliability_score=3, is_volatile=True),
_base("robtex_ip", "network",
"https://freeapi.robtex.com/ipquery/{target}", "GET",
{"as": "$.asname"},
input_type="ip", output_type=["domain"],
tags=["passive", "fast"],
health_check_url="https://freeapi.robtex.com", reliability_score=4),
_base("wayback_machine", "archive",
"https://archive.org/wayback/available?url={target}", "GET",
{"snapshot": "$.archived_snapshots.closest.url"},
input_type="url", output_type=["url"],
tags=["passive"],
health_check_url="https://archive.org", reliability_score=5),
_base("ipvigilante", "geolocation",
"https://ipvigilante.com/json/{target}", "GET",
{"city": "$.data.city_name"},
input_type="ip", output_type=["ip"],
tags=["passive"],
health_check_url="https://ipvigilante.com", reliability_score=3, is_volatile=True),
_base("pypi_user", "social",
"https://pypi.org/pypi", "POST",
{},
headers={"Content-Type": "text/xml"},
input_type="username", output_type=["username"],
normalization_map={},
regex_pattern=r"Owner\s*([^<]+)",
tags=["passive"],
health_check_url="https://pypi.org", reliability_score=4,
raw_payload="user_packages{target}"),
_base("npm_user", "social",
"https://registry.npmjs.org/-/v1/search?text=maintainer:{target}", "GET",
{"packages": "$.objects[*].package.name"},
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://registry.npmjs.org", reliability_score=5),
_base("gitlab_search", "social",
"https://gitlab.com/api/v4/users?username={target}", "GET",
{"id": "$.[*].id"},
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://gitlab.com", reliability_score=5),
_base("hackernews_user", "social",
"https://hacker-news.firebaseio.com/v0/user/{target}.json", "GET",
{"karma": "$.karma"},
input_type="username", output_type=["username"],
tags=["passive", "fast"],
health_check_url="https://hacker-news.firebaseio.com", reliability_score=5),
_base("scamwatcher", "threat_intel",
"https://www.scamwatcher.com/scam/search?q={target}", "GET",
{"results": "text_lines"},
headers={"User-Agent": "Mozilla/5.0"},
input_type="any", output_type=["domain"],
tags=["passive", "threat"],
health_check_url="https://www.scamwatcher.com", reliability_score=2, is_volatile=True),
_base("phishtank_check", "threat_intel",
"https://checkurl.phishtank.com/checkurl/", "POST",
{"in_database": "$.results.in_database"},
input_type="url", output_type=["url"],
payload_template={"url": "{target}", "format": "json"},
tags=["passive", "threat"],
health_check_url="https://checkurl.phishtank.com", reliability_score=4),
_base("duckduckgo_api", "search",
"https://search.sapti.me/search?q={target}&format=json&categories=general", "GET",
{"results": "$.results"},
input_type="any", output_type=["url"],
normalization_map={"url": "url", "title": "title"},
tags=["passive", "fast"],
backup_endpoints=[
"https://searx.tiekoetter.com/search?q={target}&format=json&categories=general",
"https://searx.perennialte.ch/search?q={target}&format=json&categories=general",
"https://search.mdosch.de/search?q={target}&format=json&categories=general",
"https://paulgo.io/search?q={target}&format=json&categories=general",
"https://priv.au/search?q={target}&format=json&categories=general",
],
health_check_url="https://search.sapti.me", reliability_score=3, is_volatile=True),
_base("cve_search", "vulns",
"https://cve.circl.lu/api/cve/{target}", "GET",
{"summary": "$.summary"},
input_type="cve", output_type=["cve"],
normalization_map={"summary": "vuln_description"},
tags=["passive"],
health_check_url="https://cve.circl.lu", reliability_score=4),
_base("cxsecurity", "vulns",
"https://cxsecurity.com/cvejson.php?cve={target}", "GET",
{"title": "$.title"},
input_type="cve", output_type=["cve"],
tags=["passive"],
health_check_url="https://cxsecurity.com", reliability_score=3, is_volatile=True),
_base("packetstorm", "vulns",
"https://packetstormsecurity.com/search/?q={target}", "GET",
{"results": "text_lines"},
input_type="any", output_type=["url"],
tags=["passive"],
health_check_url="https://packetstormsecurity.com", reliability_score=4),
_base("proxynova_comb", "breaches",
"https://api.proxynova.com/comb?query={target}", "GET",
{"lines": "$.lines"},
input_type="email", output_type=["email"],
normalization_map={"lines": "credential_line"},
tags=["passive", "stealth"],
health_check_url="https://api.proxynova.com", reliability_score=3, is_volatile=True),
_base("shodan_internetdb", "scanners",
"https://internetdb.shodan.io/{target}", "GET",
{"hostnames": "$.hostnames", "ports": "$.ports", "vulns": "$.vulns"},
input_type="ip", output_type=["domain", "ip"],
normalization_map={"hostnames": "domain", "vulns": "cve"},
tags=["passive", "fast", "infrastructure"],
health_check_url="https://internetdb.shodan.io",
reliability_score=5),
_base("circl_hashlookup", "hashes",
"https://hashlookup.circl.lu/lookup/md5/{target}", "GET",
{"filename": "$.FileName", "known_malicious": "$.KnownMalicious"},
input_type="hash", output_type=["hash"],
normalization_map={"FileName": "filename", "MD5": "hash_md5"},
tags=["passive", "fast"],
health_check_url="https://hashlookup.circl.lu",
reliability_score=5,
backup_endpoints=[
"https://hashlookup.circl.lu/lookup/sha1/{target}",
"https://hashlookup.circl.lu/lookup/sha256/{target}",
]),
_base("ipapi_is", "geolocation",
"https://api.ipapi.is/?q={target}", "GET",
{"org": "$.org", "asn": "$.asn.asn", "abuse": "$.abuse.email"},
input_type="ip", output_type=["domain"],
normalization_map={"org": "asn_org", "asn": "asn_number", "abuse": "abuse_contact"},
tags=["passive", "fast"],
health_check_url="https://api.ipapi.is",
reliability_score=4),
]
# ---------------------------------------------------------------------------
# AUTHENTICATED / PREMIUM SOURCES
# ---------------------------------------------------------------------------
AUTHENTICATED_PREMIUM_SOURCES: List[SourceConfig] = [
# ── Scanners ─────────────────────────────────────────────────────────────
_auth("shodan_host", "scanners",
"https://api.shodan.io/shodan/host/{target}?key={SHODAN_API_KEY}", "GET",
{"ports": "$.ports", "vulns": "$.vulns"},
api_key_slots=["{SHODAN_API_KEY}"],
input_type="ip", output_type=["domain", "vuln"],
normalization_map={"ports": "open_ports", "vulns": "cve_list"},
tags=["passive", "infrastructure"],
health_check_url="https://api.shodan.io", reliability_score=5),
_auth("shodan_search", "scanners",
"https://api.shodan.io/shodan/host/search?key={SHODAN_API_KEY}&query={target}", "GET",
{"ips": "$.matches[*].ip_str"},
api_key_slots=["{SHODAN_API_KEY}"],
input_type="domain", output_type=["ip"],
normalization_map={"ip_str": "ip_address"},
tags=["passive", "infrastructure"],
health_check_url="https://api.shodan.io", reliability_score=5),
_auth("shodan_dns", "dns_recon",
"https://api.shodan.io/dns/domain/{target}?key={SHODAN_API_KEY}", "GET",
{"subdomains": "$.subdomains"},
api_key_slots=["{SHODAN_API_KEY}"],
input_type="domain", output_type=["domain"],
tags=["passive", "infrastructure"],
health_check_url="https://api.shodan.io", reliability_score=5),
_auth("shodan_exploits", "vulns",
"https://exploits.shodan.io/api/search?query={target}&key={SHODAN_API_KEY}", "GET",
{"total": "$.total"},
api_key_slots=["{SHODAN_API_KEY}"],
input_type="cve", output_type=["cve"],
tags=["passive"],
health_check_url="https://exploits.shodan.io", reliability_score=5),
_auth("censys_hosts", "scanners",
"https://search.censys.io/api/v2/hosts/search?q={target}", "GET",
{"results": "$.result.hits[*].ip"},
headers={"Authorization": "Basic {CENSYS_AUTH_BASE64}"},
api_key_slots=["{CENSYS_AUTH_BASE64}"],
input_type="domain", output_type=["ip"],
normalization_map={"ip": "ip_address"},
tags=["passive", "infrastructure"],
health_check_url="https://search.censys.io", reliability_score=5),
_auth("binaryedge_exposed", "scanners",
"https://api.binaryedge.io/v2/query/ip/{target}", "GET",
{"ports": "$.events[*].port"},
headers={"X-Key": "{BINARYEDGE_API_KEY}"},
api_key_slots=["{BINARYEDGE_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"port": "open_port"},
tags=["passive", "infrastructure"],
health_check_url="https://api.binaryedge.io", reliability_score=4),
_auth("binaryedge_dns", "dns_recon",
"https://api.binaryedge.io/v2/query/domains/subdomain/{target}", "GET",
{"subs": "$.subs"},
headers={"X-Key": "{BINARYEDGE_API_KEY}"},
api_key_slots=["{BINARYEDGE_API_KEY}"],
input_type="domain", output_type=["domain"],
tags=["passive"],
health_check_url="https://api.binaryedge.io", reliability_score=4),
_auth("zoomeye_host", "scanners",
"https://api.zoomeye.org/host/search?query={target}", "GET",
{"hosts": "$.matches[*].ip"},
headers={"API-KEY": "{ZOOMEYE_API_KEY}"},
api_key_slots=["{ZOOMEYE_API_KEY}"],
input_type="domain", output_type=["ip"],
tags=["passive", "infrastructure"],
health_check_url="https://api.zoomeye.org", reliability_score=4),
_auth("fofa_info", "scanners",
"https://fofa.info/api/v1/search/all?email={FOFA_EMAIL}&key={FOFA_API_KEY}&qbase64={target}", "GET",
{"results": "$.results"},
api_key_slots=["{FOFA_API_KEY}", "{FOFA_EMAIL}"],
input_type="domain", output_type=["ip", "domain"],
query_transform="fofa_domain",
tags=["passive", "infrastructure"],
health_check_url="https://fofa.info", reliability_score=4),
_auth("onyphe_datascan", "scanners",
"https://www.onyphe.io/api/v2/simple/datascan/{target}", "GET",
{"results": "$.results"},
headers={"Authorization": "apikey {ONYPHE_API_KEY}"},
api_key_slots=["{ONYPHE_API_KEY}"],
input_type="ip", output_type=["ip", "domain"],
tags=["passive", "infrastructure"],
health_check_url="https://www.onyphe.io", reliability_score=4),
_auth("criminalip_asset", "scanners",
"https://api.criminalip.io/v1/asset/ip/report?ip={target}", "GET",
{"score": "$.score"},
headers={"x-api-key": "{CRIMINALIP_API_KEY}"},
api_key_slots=["{CRIMINALIP_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"score": "risk_score"},
tags=["passive", "threat"],
health_check_url="https://api.criminalip.io", reliability_score=4),
# ── Threat Intel ─────────────────────────────────────────────────────────
_auth("virustotal_domain", "threat_intel",
"https://www.virustotal.com/api/v3/domains/{target}", "GET",
{"malicious": "$.data.attributes.last_analysis_stats.malicious"},
rate_limit=15.0, headers={"x-apikey": "{VIRUSTOTAL_API_KEY}"},
api_key_slots=["{VIRUSTOTAL_API_KEY}"],
input_type="domain", output_type=["domain"],
normalization_map={"malicious": "malicious_count"},
tags=["passive", "threat"],
health_check_url="https://www.virustotal.com", reliability_score=5),
_auth("virustotal_ip", "threat_intel",
"https://www.virustotal.com/api/v3/ip_addresses/{target}", "GET",
{"reputation": "$.data.attributes.reputation"},
rate_limit=15.0, headers={"x-apikey": "{VIRUSTOTAL_API_KEY}"},
api_key_slots=["{VIRUSTOTAL_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"reputation": "vt_reputation"},
tags=["passive", "threat"],
health_check_url="https://www.virustotal.com", reliability_score=5),
_auth("greynoise_community", "threat_intel",
"https://api.greynoise.io/v3/community/{target}", "GET",
{"noise": "$.noise", "classification": "$.classification"},
headers={"key": "{GREYNOISE_API_KEY}"},
api_key_slots=["{GREYNOISE_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"noise": "is_noise", "classification": "threat_class"},
tags=["passive", "threat"],
health_check_url="https://api.greynoise.io", reliability_score=5),
_auth("abuseipdb", "threat_intel",
"https://api.abuseipdb.com/api/v2/check?ipAddress={target}", "GET",
{"score": "$.data.abuseConfidenceScore"},
headers={"Key": "{ABUSEIPDB_API_KEY}"},
api_key_slots=["{ABUSEIPDB_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"abuseConfidenceScore": "abuse_score"},
tags=["passive", "threat"],
health_check_url="https://api.abuseipdb.com", reliability_score=5),
_auth("pulsedive_analyze", "threat_intel",
"https://pulsedive.com/api/analyze.php?value={target}", "GET",
{"risk": "$.risk"},
headers={"key": "{PULSEDIVE_API_KEY}"},
api_key_slots=["{PULSEDIVE_API_KEY}"],
input_type="any", output_type=["ip", "domain"],
normalization_map={"risk": "risk_level"},
tags=["passive", "threat"],
health_check_url="https://pulsedive.com", reliability_score=4),
_auth("metadefender_ip", "threat_intel",
"https://api.metadefender.com/v4/ip/{target}", "GET",
{"lookup": "$.lookup_results"},
headers={"apikey": "{METADEFENDER_API_KEY}"},
api_key_slots=["{METADEFENDER_API_KEY}"],
input_type="ip", output_type=["ip"],
tags=["passive", "threat"],
health_check_url="https://api.metadefender.com", reliability_score=4),
_auth("recordedfuture_ip", "threat_intel",
"https://api.recordedfuture.com/v2/ip/{target}", "GET",
{"risk": "$.data.risk.score"},
headers={"X-RFToken": "{RF_TOKEN}"},
api_key_slots=["{RF_TOKEN}"],
input_type="ip", output_type=["ip"],
normalization_map={"score": "rf_risk_score"},
tags=["passive", "threat"],
health_check_url="https://api.recordedfuture.com", reliability_score=5),
_auth("vulners_search", "threat_intel",
"https://vulners.com/api/v3/search/lucene/?query={target}", "GET",
{"results": "$.data.search[*]._source.title"},
headers={"X-Vulners-Api-Key": "{VULNERS_API_KEY}"},
api_key_slots=["{VULNERS_API_KEY}"],
input_type="cve", output_type=["cve"],
tags=["passive"],
health_check_url="https://vulners.com", reliability_score=4),
_auth("urlvoid", "threat_intel",
"https://api.urlvoid.com/api1000/{URLVOID_API_KEY}/host/{target}", "GET",
{"detections": "$.detections"},
api_key_slots=["{URLVOID_API_KEY}"],
input_type="domain", output_type=["domain"],
tags=["passive", "threat"],
health_check_url="https://api.urlvoid.com", reliability_score=4),
_auth("fraudlabspro", "threat_intel",
"https://api.fraudlabspro.com/v1/ip/check?key={FLP_API_KEY}&ip={target}", "GET",
{"fraud": "$.fraudlabspro_score"},
api_key_slots=["{FLP_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"fraudlabspro_score": "fraud_score"},
tags=["passive", "threat"],
health_check_url="https://api.fraudlabspro.com", reliability_score=4),
_auth("google_safebrowsing", "threat_intel",
"https://safebrowsing.googleapis.com/v4/threatMatches:find?key={GOOGLE_API_KEY}", "POST",
{"matches": "$.matches"},
api_key_slots=["{GOOGLE_API_KEY}"],
input_type="url", output_type=["url"],
payload_template={"client": {"clientId": "nox", "clientVersion": "1.0"},
"threatInfo": {"threatTypes": ["MALWARE", "SOCIAL_ENGINEERING"],
"platformTypes": ["ANY_PLATFORM"],
"threatEntryTypes": ["URL"],
"threatEntries": [{"url": "{target}"}]}},
tags=["passive", "threat"],
health_check_url="https://safebrowsing.googleapis.com", reliability_score=5),
_auth("threatconnect_search", "threat_intel",
"https://api.threatconnect.com/v2/indicators/{target}", "GET",
{"data": "$.data"},
headers={"Authorization": "TC {TC_API_KEY}"},
api_key_slots=["{TC_API_KEY}"],
input_type="any", output_type=["ip", "domain"],
tags=["passive", "threat"],
health_check_url="https://api.threatconnect.com", reliability_score=2, is_volatile=True),
_auth("threatportal", "threat_intel",
"https://threatportal.io/api/v1/search?q={target}", "GET",
{"results": "$.results"},
headers={"Authorization": "Bearer {TP_API_KEY}"},
api_key_slots=["{TP_API_KEY}"],
input_type="any", output_type=["ip", "domain"],
tags=["passive", "threat"],
health_check_url="https://threatportal.io", reliability_score=3, is_volatile=True),
_auth("malshare", "threat_intel",
"https://malshare.com/api.php?api_key={MALSHARE_API_KEY}&action=search&query={target}", "GET",
{"hashes": "$.*"},
api_key_slots=["{MALSHARE_API_KEY}"],
input_type="hash", output_type=["hash"],
tags=["passive", "threat"],
health_check_url="https://malshare.com", reliability_score=3),
_auth("hybrid_analysis", "threat_intel",
"https://www.hybrid-analysis.com/api/v2/search/hash", "POST",
{"verdict": "$.verdict"},
headers={"api-key": "{HYBRID_API_KEY}"},
api_key_slots=["{HYBRID_API_KEY}"],
input_type="hash", output_type=["hash"],
payload_template={"hash": "{target}"},
normalization_map={"verdict": "malware_verdict"},
tags=["passive", "threat", "heavy"],
health_check_url="https://www.hybrid-analysis.com", reliability_score=4),
_auth("joesandbox", "threat_intel",
"https://www.joesandbox.com/api/v2/analysis/search?q={target}", "GET",
{"id": "$.[*].id"},
headers={"X-JoeSandbox-Api-Key": "{JOE_API_KEY}"},
api_key_slots=["{JOE_API_KEY}"],
input_type="hash", output_type=["hash"],
tags=["passive", "threat", "heavy"],
health_check_url="https://www.joesandbox.com", reliability_score=4),
_auth("anyrun", "threat_intel",
"https://api.any.run/v1/analysis?hash={target}", "GET",
{"tasks": "$.tasks"},
headers={"Authorization": "API-Key {ANYRUN_API_KEY}"},
api_key_slots=["{ANYRUN_API_KEY}"],
input_type="hash", output_type=["hash"],
tags=["passive", "threat", "heavy"],
health_check_url="https://api.any.run", reliability_score=4),
_auth("intezer", "threat_intel",
"https://analyze.intezer.com/api/v2-0/get-analysis-by-hash/{target}", "GET",
{"result": "$.result"},
headers={"Authorization": "Bearer {INTEZER_API_KEY}"},
api_key_slots=["{INTEZER_API_KEY}"],
input_type="hash", output_type=["hash"],
tags=["passive", "threat"],
health_check_url="https://analyze.intezer.com", reliability_score=4),
_auth("misp_search", "threat_intel",
"{MISP_URL}/attributes/restSearch", "POST",
{"attributes": "$.Attribute[*].value"},
headers={"Authorization": "{MISP_API_KEY}", "Content-Type": "application/json"},
api_key_slots=["{MISP_API_KEY}", "{MISP_URL}"],
input_type="any", output_type=["ip", "domain", "hash"],
payload_template={"returnFormat": "json", "value": "{target}"},
tags=["passive", "threat"],
health_check_url="https://misp.local", reliability_score=4),
]
AUTHENTICATED_PREMIUM_SOURCES += [
# ── Breaches ─────────────────────────────────────────────────────────────
_auth("hibp_breached", "breaches",
"https://haveibeenpwned.com/api/v3/breachedaccount/{target}", "GET",
{"breaches": "$.*.Name"},
rate_limit=1.5,
headers={"hibp-api-key": "{HIBP_API_KEY}", "User-Agent": "NOX-Framework"},
api_key_slots=["{HIBP_API_KEY}"],
input_type="email", output_type=["email", "domain"],
normalization_map={"Name": "breach_name"},
tags=["passive", "stealth"],
health_check_url="https://haveibeenpwned.com", reliability_score=5),
_auth("dehashed", "breaches",
"https://api.dehashed.com/v2/search?query={target}", "GET",
{"entries": "$.entries"},
headers={"Authorization": "Basic {DEHASHED_AUTH_BASE64}", "Accept": "application/json"},
api_key_slots=["{DEHASHED_AUTH_BASE64}"],
input_type="email", output_type=["email", "username", "ip"],
normalization_map={"email": "email_address", "username": "username",
"password": "plaintext_password", "hashed_password": "password_hash",
"ip_address": "ip_address", "name": "full_name"},
tags=["passive", "stealth"],
health_check_url="https://api.dehashed.com", reliability_score=5),
_auth("snusbase", "breaches",
"https://api.snusbase.com/data/search", "POST",
{"leaks": "$.results"},
headers={"Auth": "{SNUSBASE_API_KEY}", "Content-Type": "application/json"},
api_key_slots=["{SNUSBASE_API_KEY}"],
input_type="email", output_type=["email", "username"],
payload_template={"terms": ["{target}"], "types": ["email"]},
normalization_map={"email": "email_address", "username": "username",
"password": "plaintext_password", "hash": "password_hash"},
tags=["passive", "stealth"],
health_check_url="https://api.snusbase.com", reliability_score=4),
_auth("intelx_search", "breaches",
"https://2.intelx.io/intelligent/search", "POST",
{"id": "$.id"},
headers={"x-key": "{INTELX_API_KEY}"},
api_key_slots=["{INTELX_API_KEY}"],
input_type="email", output_type=["email", "domain"],
payload_template={"term": "{target}", "buckets": [], "lookuplevel": 0,
"maxresults": 100, "timeout": 0, "datefrom": "", "dateto": "",
"sort": 4, "media": 0, "terminate": []},
poll_endpoint="https://2.intelx.io/intelligent/search/result",
poll_id_field="id",
poll_id_param="id",
poll_json_root="records",
tags=["passive", "stealth"],
health_check_url="https://2.intelx.io", reliability_score=5),
_auth("intelx_phone", "breaches",
"https://2.intelx.io/phone/search?phone={target}", "GET",
{"results": "$.results"},
headers={"x-key": "{INTELX_API_KEY}"},
api_key_slots=["{INTELX_API_KEY}"],
input_type="phone", output_type=["phone"],
tags=["passive"],
health_check_url="https://2.intelx.io", reliability_score=5),
_auth("leakcheck", "breaches",
"https://leakcheck.io/api/v2/query/{target}", "GET",
{"sources": "$.sources"},
headers={"X-API-Key": "{LEAKCHECK_API_KEY}"},
api_key_slots=["{LEAKCHECK_API_KEY}"],
input_type="email", output_type=["email"],
normalization_map={"sources": "breach_sources"},
tags=["passive", "stealth"],
health_check_url="https://leakcheck.io", reliability_score=4),
_auth("spycloud_breach", "breaches",
"https://api.spycloud.io/enterprise-v2/breach/catalog/emails/{target}", "GET",
{"results": "$.results"},
headers={"X-API-Key": "{SPYCLOUD_API_KEY}"},
api_key_slots=["{SPYCLOUD_API_KEY}"],
input_type="email", output_type=["email", "username", "ip"],
normalization_map={"email": "email_address", "username": "username",
"password": "plaintext_password", "ip_addresses": "ip_address"},
tags=["passive", "stealth"],
health_check_url="https://api.spycloud.io", reliability_score=5),
_auth("leakix_search", "breaches",
"https://leakix.net/api/search?q={target}", "GET",
{"leaks": "$.[*].event_source"},
headers={"api-key": "{LEAKIX_API_KEY}"},
api_key_slots=["{LEAKIX_API_KEY}"],
input_type="domain", output_type=["domain", "ip"],
tags=["passive"],
health_check_url="https://leakix.net", reliability_score=4),
_auth("breachdirectory", "breaches",
"https://breachdirectory.com/api/search?key={BD_API_KEY}&email={target}", "GET",
{"found": "$.found"},
api_key_slots=["{BD_API_KEY}"],
input_type="email", output_type=["email"],
tags=["passive", "stealth"],
health_check_url="https://breachdirectory.com", reliability_score=4),
_auth("breachaware", "breaches",
"https://api.breachaware.com/v1/search?query={target}", "GET",
{"breaches": "$.breaches"},
headers={"X-API-KEY": "{BA_API_KEY}"},
api_key_slots=["{BA_API_KEY}"],
input_type="email", output_type=["email"],
tags=["passive", "stealth"],
health_check_url="https://api.breachaware.com", reliability_score=3, is_volatile=True),
_auth("flare_leaksdb", "breaches",
"https://api.flare.io/leaksdb/v2/credentials/email/{target}", "GET",
{"items": "$.items"},
headers={"Authorization": "Bearer {FLARE_API_KEY}"},
api_key_slots=["{FLARE_API_KEY}"],
input_type="email", output_type=["email", "username"],
normalization_map={"email": "email_address", "username": "username",
"password": "plaintext_password", "hash": "password_hash",
"source": "breach_name"},
tags=["passive", "stealth"],
health_check_url="https://api.flare.io", reliability_score=4),
_auth("leak_lookup", "breaches",
"https://leak-lookup.com/api/search", "POST",
{"results": "$.message"},
headers={"X-API-Key": "{LEAK_LOOKUP_API_KEY}"},
api_key_slots=["{LEAK_LOOKUP_API_KEY}"],
input_type="email", output_type=["email"],
payload_template={"query": "{target}", "type": "email_address"},
tags=["passive", "stealth"],
health_check_url="https://leak-lookup.com", reliability_score=3, is_volatile=True),
# ── DNS Recon ─────────────────────────────────────────────────────────────
_auth("securitytrails_sub", "dns_recon",
"https://api.securitytrails.com/v1/domain/{target}/subdomains", "GET",
{"subdomains": "$.subdomains"},
headers={"APIKEY": "{SECURITYTRAILS_API_KEY}"},
api_key_slots=["{SECURITYTRAILS_API_KEY}"],
input_type="domain", output_type=["domain"],
tags=["passive"],
health_check_url="https://api.securitytrails.com", reliability_score=5),
_auth("securitytrails_history", "dns_recon",
"https://api.securitytrails.com/v1/history/{target}/dns/a", "GET",
{"history": "$.records[*].values[*].ip"},
headers={"APIKEY": "{SECURITYTRAILS_API_KEY}"},
api_key_slots=["{SECURITYTRAILS_API_KEY}"],
input_type="domain", output_type=["ip"],
normalization_map={"ip": "historical_ip"},
tags=["passive"],
health_check_url="https://api.securitytrails.com", reliability_score=5),
_auth("circl_lu_pdns", "dns_recon",
"https://www.circl.lu/pdns/query/{target}", "GET",
{"resolutions": "$.[*].rdata"},
headers={"Authorization": "Basic {CIRCL_AUTH_BASE64}"},
api_key_slots=["{CIRCL_AUTH_BASE64}"],
input_type="domain", output_type=["ip"],
tags=["passive"],
health_check_url="https://www.circl.lu", reliability_score=4),
_auth("viewdns_reverse_ip", "dns_recon",
"https://api.viewdns.info/reverseip/?host={target}&apikey={VIEWDNS_API_KEY}&output=json", "GET",
{"domains": "$.response.domains[*].name"},
api_key_slots=["{VIEWDNS_API_KEY}"],
input_type="ip", output_type=["domain"],
tags=["passive"],
health_check_url="https://api.viewdns.info", reliability_score=4),
_auth("dnsdb_pdns", "dns_recon",
"https://api.dnsdb.info/lookup/rrset/name/{target}", "GET",
{"rdata": "$.[*].rdata"},
headers={"X-API-Key": "{DNSDB_API_KEY}"},
api_key_slots=["{DNSDB_API_KEY}"],
input_type="domain", output_type=["ip"],
tags=["passive"],
health_check_url="https://api.dnsdb.info", reliability_score=5),
_auth("spyonweb", "dns_recon",
"https://api.spyonweb.com/v1/summary/{target}?access_token={SPYONWEB_API_KEY}", "GET",
{"adsense": "$.result.adsense"},
api_key_slots=["{SPYONWEB_API_KEY}"],
input_type="domain", output_type=["domain"],
tags=["passive"],
health_check_url="https://api.spyonweb.com", reliability_score=1,
is_volatile=True, confidence=0.1),
# ── WHOIS ─────────────────────────────────────────────────────────────────
_auth("passivetotal_whois", "whois",
"https://api.passivetotal.org/v2/whois?query={target}", "GET",
{"registrar": "$.registrar"},
headers={"Authorization": "Basic {PASSIVETOTAL_AUTH_BASE64}"},
api_key_slots=["{PASSIVETOTAL_AUTH_BASE64}"],
input_type="domain", output_type=["email", "domain"],
normalization_map={"registrar": "registrar_name"},
tags=["passive"],
health_check_url="https://api.passivetotal.org", reliability_score=4),
_auth("whoisxml_api", "whois",
"https://www.whoisxmlapi.com/whoisserver/WhoisService?apiKey={WHOISXML_API_KEY}&domainName={target}&outputFormat=JSON", "GET",
{"created": "$.WhoisRecord.createdDate"},
api_key_slots=["{WHOISXML_API_KEY}"],
input_type="domain", output_type=["email", "domain"],
normalization_map={"createdDate": "registration_date"},
tags=["passive"],
health_check_url="https://www.whoisxmlapi.com", reliability_score=5),
_auth("whoxy_whois", "whois",
"https://api.whoxy.com/?key={WHOXY_API_KEY}&whois={target}", "GET",
{"registrar": "$.registrar_name"},
api_key_slots=["{WHOXY_API_KEY}"],
input_type="domain", output_type=["email", "domain"],
tags=["passive"],
health_check_url="https://api.whoxy.com", reliability_score=4),
_auth("whois_freaks", "whois",
"https://whoisfreaks.com/api/v1/whois?apiKey={WF_API_KEY}&whois=live&domainName={target}", "GET",
{"emails": "$.whois_record.registrant_contact.email_address"},
api_key_slots=["{WF_API_KEY}"],
input_type="domain", output_type=["email"],
tags=["passive"],
health_check_url="https://whoisfreaks.com", reliability_score=4),
_auth("domaintools_whois", "whois",
"https://api.domaintools.com/v1/{target}/whois/", "GET",
{"whois": "$.response.whois.record"},
headers={"Authorization": "Basic {DT_AUTH_BASE64}"},
api_key_slots=["{DT_AUTH_BASE64}"],
input_type="domain", output_type=["email", "domain"],
tags=["passive"],
health_check_url="https://api.domaintools.com", reliability_score=5),
# ── Enrichment ────────────────────────────────────────────────────────────
_auth("seon_email", "enrichment",
"https://api.seon.io/SeonRestService/email-api/v3", "POST",
{"data": "$.data"},
headers={"X-API-KEY": "{SEON_API_KEY}", "Content-Type": "application/json"},
api_key_slots=["{SEON_API_KEY}"],
input_type="email", output_type=["email", "domain", "username"],
payload_template={"email": "{target}"},
normalization_map={"email": "email_address", "domain": "domain",
"full_name": "full_name", "phone_number": "phone"},
tags=["passive"],
health_check_url="https://api.seon.io", reliability_score=4),
_auth("fullcontact", "enrichment",
"https://api.fullcontact.com/v3/person.enrich", "POST",
{"social": "$.socialProfiles"},
headers={"Authorization": "Bearer {FULLCONTACT_API_KEY}"},
api_key_slots=["{FULLCONTACT_API_KEY}"],
input_type="email", output_type=["username", "domain"],
payload_template={"email": "{target}"},
tags=["passive"],
health_check_url="https://api.fullcontact.com", reliability_score=4),
_auth("passivetotal_enrich", "enrichment",
"https://api.passivetotal.org/v2/enrichment?query={target}", "GET",
{"tags": "$.tags"},
headers={"Authorization": "Basic {PASSIVETOTAL_AUTH_BASE64}"},
api_key_slots=["{PASSIVETOTAL_AUTH_BASE64}"],
input_type="domain", output_type=["domain"],
tags=["passive"],
health_check_url="https://api.passivetotal.org", reliability_score=4),
_auth("pipl_search", "enrichment",
"https://api.pipl.com/search/?email={target}&key={PIPL_API_KEY}", "GET",
{"person": "$.person"},
api_key_slots=["{PIPL_API_KEY}"],
input_type="email", output_type=["username", "domain", "phone"],
tags=["passive"],
health_check_url="https://api.pipl.com", reliability_score=2,
is_volatile=True, confidence=0.3),
# ── Email Reputation ──────────────────────────────────────────────────────
_auth("ipqualityscore_email", "email_rep",
"https://ipqualityscore.com/api/json/email/{IPQS_API_KEY}/{target}", "GET",
{"fraud_score": "$.fraud_score"},
api_key_slots=["{IPQS_API_KEY}"],
input_type="email", output_type=["email"],
normalization_map={"fraud_score": "email_fraud_score"},
tags=["passive", "fast"],
health_check_url="https://ipqualityscore.com", reliability_score=4),
_auth("emailhippo", "email_rep",
"https://api.emailhippo.com/v3/verify?apiKey={HIPPO_API_KEY}&email={target}", "GET",
{"status": "$.meta.status"},
api_key_slots=["{HIPPO_API_KEY}"],
input_type="email", output_type=["email"],
tags=["passive", "fast"],
health_check_url="https://api.emailhippo.com", reliability_score=4),
_auth("zerobounce", "email_rep",
"https://api.zerobounce.net/v2/validate?api_key={ZEROBOUNCE_API_KEY}&email={target}", "GET",
{"status": "$.status"},
api_key_slots=["{ZEROBOUNCE_API_KEY}"],
input_type="email", output_type=["email"],
normalization_map={"status": "email_validity"},
tags=["passive", "fast"],
health_check_url="https://api.zerobounce.net", reliability_score=4),
_auth("hunter_verify", "email_rep",
"https://api.hunter.io/v2/email-verifier?email={target}&api_key={HUNTER_API_KEY}", "GET",
{"result": "$.data.result"},
api_key_slots=["{HUNTER_API_KEY}"],
input_type="email", output_type=["email"],
tags=["passive", "fast"],
health_check_url="https://api.hunter.io", reliability_score=4),
_auth("mailboxlayer", "email_rep",
"https://apilayer.net/api/check?access_key={MAILBOX_API_KEY}&email={target}", "GET",
{"score": "$.score"},
api_key_slots=["{MAILBOX_API_KEY}"],
input_type="email", output_type=["email"],
tags=["passive"],
health_check_url="https://apilayer.net", reliability_score=3),
_auth("abstract_email", "email_rep",
"https://emailvalidation.abstractapi.com/v1/?api_key={ABSTRACT_API_KEY}&email={target}", "GET",
{"quality": "$.quality_score"},
api_key_slots=["{ABSTRACT_API_KEY}"],
input_type="email", output_type=["email"],
tags=["passive", "fast"],
health_check_url="https://emailvalidation.abstractapi.com", reliability_score=4),
# ── Discovery / Social ────────────────────────────────────────────────────
_auth("hunter_io", "discovery",
"https://api.hunter.io/v2/domain-search?domain={target}&api_key={HUNTER_API_KEY}", "GET",
{"emails": "$.data.emails[*].value"},
api_key_slots=["{HUNTER_API_KEY}"],
input_type="domain", output_type=["email"],
normalization_map={"value": "email_address"},
tags=["passive"],
health_check_url="https://api.hunter.io", reliability_score=5),
_auth("twitter_v2", "social",
"https://api.twitter.com/2/users/by/username/{target}", "GET",
{"id": "$.data.id"},
headers={"Authorization": "Bearer {TWITTER_BEARER_TOKEN}"},
api_key_slots=["{TWITTER_BEARER_TOKEN}"],
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://api.twitter.com", reliability_score=1,
is_volatile=True, confidence=0.1),
_auth("github_code_search", "code",
"https://api.github.com/search/code?q={target}", "GET",
{"urls": "$.items[*].html_url"},
headers={"Authorization": "token {GITHUB_TOKEN}"},
api_key_slots=["{GITHUB_TOKEN}"],
input_type="any", output_type=["url"],
tags=["passive"],
health_check_url="https://api.github.com", reliability_score=5),
_auth("github_search_repos", "social",
"https://api.github.com/search/repositories?q={target}", "GET",
{"total": "$.total_count"},
headers={"Authorization": "token {GITHUB_TOKEN}"},
api_key_slots=["{GITHUB_TOKEN}"],
input_type="username", output_type=["username"],
tags=["passive"],
health_check_url="https://api.github.com", reliability_score=5),
# ── Geolocation ───────────────────────────────────────────────────────────
_auth("ipstack", "geolocation",
"https://api.ipstack.com/{target}?access_key={IPSTACK_API_KEY}", "GET",
{"country": "$.country_name"},
api_key_slots=["{IPSTACK_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"country_name": "geo_country"},
tags=["passive", "fast"],
health_check_url="https://api.ipstack.com", reliability_score=4),
_auth("ipgeolocation_io", "geolocation",
"https://api.ipgeolocation.io/ipgeo?apiKey={IPGEO_API_KEY}&ip={target}", "GET",
{"isp": "$.isp"},
api_key_slots=["{IPGEO_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"isp": "asn_org"},
tags=["passive", "fast"],
health_check_url="https://api.ipgeolocation.io", reliability_score=4),
_auth("ipdata_co", "geolocation",
"https://api.ipdata.co/{target}?api-key={IPDATA_API_KEY}", "GET",
{"threat": "$.threat"},
api_key_slots=["{IPDATA_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"threat": "threat_info"},
tags=["passive", "fast"],
health_check_url="https://api.ipdata.co", reliability_score=4),
_auth("extreme_ip_lookup", "geolocation",
"https://extreme-ip-lookup.com/json/{target}?key={EXTREME_API_KEY}", "GET",
{"org": "$.org"},
api_key_slots=["{EXTREME_API_KEY}"],
input_type="ip", output_type=["ip"],
tags=["passive"],
health_check_url="https://extreme-ip-lookup.com", reliability_score=3),
_auth("ipinfodb", "geolocation",
"https://api.ipinfodb.com/v3/ip-city/?key={IPINFODB_API_KEY}&ip={target}&format=json", "GET",
{"city": "$.cityName"},
api_key_slots=["{IPINFODB_API_KEY}"],
input_type="ip", output_type=["ip"],
normalization_map={"cityName": "geo_city"},
tags=["passive"],
health_check_url="https://api.ipinfodb.com", reliability_score=3),
# ── Phone ─────────────────────────────────────────────────────────────────
_auth("numverify", "phone",
"https://apilayer.net/api/validate?access_key={NUMVERIFY_API_KEY}&number={target}", "GET",
{"valid": "$.valid", "carrier": "$.carrier"},
api_key_slots=["{NUMVERIFY_API_KEY}"],
input_type="phone", output_type=["phone"],
normalization_map={"valid": "phone_valid", "carrier": "phone_carrier"},
tags=["passive"],
health_check_url="https://apilayer.net", reliability_score=4),
# ── Search ────────────────────────────────────────────────────────────────
_auth("google_search_custom", "search",
"https://www.googleapis.com/customsearch/v1?key={GOOGLE_CX_KEY}&cx={GOOGLE_CX_ID}&q={target}", "GET",
{"items": "$.items[*].link"},
api_key_slots=["{GOOGLE_CX_KEY}", "{GOOGLE_CX_ID}"],
input_type="any", output_type=["url"],
tags=["passive"],
health_check_url="https://www.googleapis.com", reliability_score=5),
_auth("bing_search_api", "search",
"https://api.bing.microsoft.com/v7.0/search?q={target}", "GET",
{"urls": "$.webPages.value[*].url"},
headers={"Ocp-Apim-Subscription-Key": "{BING_API_KEY}"},
api_key_slots=["{BING_API_KEY}"],
input_type="any", output_type=["url"],
tags=["passive"],
health_check_url="https://api.bing.microsoft.com", reliability_score=5),
_auth("threatfox", "threat_intel",
"https://threatfox-api.abuse.ch/api/v1/", "POST",
{"results": "$.data"},
headers={"API-KEY": "{THREATFOX_API_KEY}", "Content-Type": "application/json"},
payload_template={"query": "search_ioc", "search_term": "{target}"},
api_key_slots=["{THREATFOX_API_KEY}"],
input_type="any", output_type=["ip", "domain", "hash"],
normalization_map={"ioc": "indicator", "malware": "malware_family"},
tags=["passive", "threat"],
health_check_url="https://threatfox-api.abuse.ch",
reliability_score=5),
_auth("urlhaus", "threat_intel",
"https://urlhaus-api.abuse.ch/v1/host/", "POST",
{"urls": "$.urls"},
headers={"Auth-Key": "{URLHAUS_API_KEY}"},
payload_template={"host": "{target}"},
api_key_slots=["{URLHAUS_API_KEY}"],
input_type="domain", output_type=["url", "domain"],
normalization_map={"url": "malware_url", "threat": "threat_type"},
tags=["passive", "threat"],
health_check_url="https://urlhaus-api.abuse.ch",
reliability_score=5),
_auth("malwarebazaar", "hashes",
"https://mb-api.abuse.ch/api/v1/", "POST",
{"data": "$.data"},
headers={"API-KEY": "{MALWAREBAZAAR_API_KEY}"},
payload_template={"query": "get_info", "hash": "{target}"},
api_key_slots=["{MALWAREBAZAAR_API_KEY}"],
input_type="hash", output_type=["hash"],
normalization_map={"file_name": "filename", "tags": "tags"},
tags=["passive", "threat"],
health_check_url="https://mb-api.abuse.ch",
reliability_score=5),
_auth("fullhunt_subdomains", "dns_recon",
"https://fullhunt.io/api/v1/domain/{target}/subdomains", "GET",
{"hosts": "$.hosts"},
headers={"X-API-KEY": "{FULLHUNT_API_KEY}"},
api_key_slots=["{FULLHUNT_API_KEY}"],
input_type="domain", output_type=["domain", "ip"],
normalization_map={"host": "domain"},
tags=["passive", "infrastructure"],
health_check_url="https://fullhunt.io",
reliability_score=4),
_auth("netlas_search", "scanners",
"https://app.netlas.io/api/responses/?q={target}&source_type=include&start=0&fields=", "GET",
{"items": "$.items"},
headers={"X-API-Key": "{NETLAS_API_KEY}"},
api_key_slots=["{NETLAS_API_KEY}"],
input_type="ip", output_type=["ip", "domain"],
normalization_map={"data.ip": "ip_address", "data.domain": "domain"},
tags=["passive", "infrastructure"],
health_check_url="https://app.netlas.io",
reliability_score=4),
]
# ---------------------------------------------------------------------------
# Builder
# ---------------------------------------------------------------------------
def build_nox_sources(output_dir: str = None) -> None:
# Resolve output_dir relative to this script's location so the command
# `python /opt/nox-cli/build_sources.py` always writes to the correct
# package sources/ directory regardless of the working directory.
if output_dir is None:
output_dir = str(Path(__file__).resolve().parent / "sources")
os.makedirs(output_dir, exist_ok=True)
all_sources: List[SourceConfig] = FREE_PUBLIC_SOURCES + AUTHENTICATED_PREMIUM_SOURCES
errors: List[str] = []
written = 0
for src in all_sources:
dest = os.path.join(output_dir, f"{src.name}.json")
payload = src.to_json()
tmp_fd, tmp_path = tempfile.mkstemp(dir=output_dir, suffix=".tmp")
try:
with os.fdopen(tmp_fd, "w", encoding="utf-8") as fh:
fh.write(payload)
os.replace(tmp_path, dest)
written += 1
except Exception as exc:
os.unlink(tmp_path)
errors.append(f"[ERROR] '{src.name}': {exc}")
print(f"Done. {written}/{len(all_sources)} source files written to '{output_dir}/'.")
if errors:
print("\nErrors:")
for e in errors:
print(" ", e)
# ── Remove orphaned JSON plugins (§2.3) ───────────────────────────
# Any .json in sources/ not produced by this build is stale and would
# be silently loaded at runtime by SourceOrchestrator. Remove it.
expected_filenames = {f"{src.name}.json" for src in all_sources}
removed = 0
for fname in os.listdir(output_dir):
if not fname.endswith(".json"):
continue
fpath = os.path.join(output_dir, fname)
if os.path.isfile(fpath) and fname not in expected_filenames:
try:
os.unlink(fpath)
print(f"[cleanup] Removed orphaned plugin: {fname}")
removed += 1
except OSError as exc:
print(f"[WARN] Could not remove orphaned plugin {fname}: {exc}")
if removed:
print(f"[cleanup] {removed} orphaned plugin(s) removed.")
# ── Seed apikeys.json (never overwrites an existing file) ──────────
if _APIKEYS_FILE is not None and _default_store is not None and _write_store is not None:
if not _APIKEYS_FILE.exists():
try:
_write_store(_default_store())
print(f"Created API key template: {_APIKEYS_FILE}")
print(f" All {len(_default_store())} private keys set to '{UNIVERSAL_PLACEHOLDER}'.")
print(" Edit that file to configure your keys before scanning.")
except Exception as exc:
print(f"[WARN] Could not create apikeys.json: {exc}")
else:
print(f"API key file already exists — not overwritten: {_APIKEYS_FILE}")
if __name__ == "__main__":
build_nox_sources()