v1.0.1 — quarterly maintenance: source audit, engine fixes, WAF hardening

This commit is contained in:
nox-project
2026-04-11 14:12:06 +02:00
parent 913e764133
commit 7febdc60f5
30 changed files with 703 additions and 449 deletions
+3 -2
View File
@@ -21,7 +21,8 @@
],
"health_check_url": "https://api.bgpview.io",
"expected_status": 200,
"reliability_score": 4,
"reliability_score": 2,
"is_volatile": true,
"backup_endpoints": [],
"confidence": 0.85
"confidence": 0.55
}
+31
View File
@@ -0,0 +1,31 @@
{
"name": "circl_hashlookup",
"category": "hashes",
"endpoint": "https://hashlookup.circl.lu/lookup/md5/{target}",
"method": "GET",
"requires_auth": false,
"selectors": {
"filename": "$.FileName",
"known_malicious": "$.KnownMalicious"
},
"rate_limit": 1.0,
"headers": {},
"api_key_slots": [],
"input_type": "hash",
"output_type": [
"hash"
],
"normalization_map": {
"FileName": "filename",
"MD5": "hash_md5"
},
"tags": [
"passive",
"fast"
],
"health_check_url": "https://hashlookup.circl.lu",
"expected_status": 200,
"reliability_score": 5,
"backup_endpoints": [],
"confidence": 1.0
}
+1 -1
View File
@@ -1,7 +1,7 @@
{
"name": "dehashed",
"category": "breaches",
"endpoint": "https://api.dehashed.com/search?query={target}",
"endpoint": "https://api.dehashed.com/v2/search?query={target}",
"method": "GET",
"requires_auth": true,
"selectors": {
+10 -6
View File
@@ -1,11 +1,11 @@
{
"name": "duckduckgo_api",
"category": "search",
"endpoint": "https://api.duckduckgo.com/?q={target}&format=json",
"endpoint": "https://searx.tiekoetter.com/search?q={target}&format=json&categories=general",
"method": "GET",
"requires_auth": false,
"selectors": {
"abstract": "$.Abstract"
"results": "$.results"
},
"rate_limit": 1.0,
"headers": {},
@@ -14,14 +14,18 @@
"output_type": [
"url"
],
"normalization_map": {},
"normalization_map": {
"url": "url",
"title": "title"
},
"tags": [
"passive",
"fast"
],
"health_check_url": "https://api.duckduckgo.com",
"health_check_url": "https://searx.tiekoetter.com",
"expected_status": 200,
"reliability_score": 5,
"reliability_score": 3,
"is_volatile": true,
"backup_endpoints": [],
"confidence": 1.0
"confidence": 0.7
}
+7 -3
View File
@@ -3,13 +3,17 @@
"category": "email_rep",
"endpoint": "https://emailrep.io/{target}",
"method": "GET",
"requires_auth": false,
"requires_auth": true,
"selectors": {
"reputation": "$.reputation"
},
"rate_limit": 2.0,
"headers": {},
"api_key_slots": [],
"headers": {
"Key": "{EMAILREP_API_KEY}"
},
"api_key_slots": [
"{EMAILREP_API_KEY}"
],
"input_type": "email",
"output_type": [
"email"
+34
View File
@@ -0,0 +1,34 @@
{
"name": "fullhunt_subdomains",
"category": "dns_recon",
"endpoint": "https://fullhunt.io/api/v1/domain/{target}/subdomains",
"method": "GET",
"requires_auth": true,
"selectors": {
"hosts": "$.hosts"
},
"rate_limit": 1.0,
"headers": {
"X-API-KEY": "{FULLHUNT_API_KEY}"
},
"api_key_slots": [
"{FULLHUNT_API_KEY}"
],
"input_type": "domain",
"output_type": [
"domain",
"ip"
],
"normalization_map": {
"host": "domain"
},
"tags": [
"passive",
"infrastructure"
],
"health_check_url": "https://fullhunt.io",
"expected_status": 200,
"reliability_score": 4,
"backup_endpoints": [],
"confidence": 0.85
}
-28
View File
@@ -1,28 +0,0 @@
{
"name": "hashes_org",
"category": "hashes",
"endpoint": "https://hashes.org/api.php?key={HASHES_API_KEY}&query={target}",
"method": "GET",
"requires_auth": true,
"selectors": {
"found": "$.results"
},
"rate_limit": 1.0,
"headers": {},
"api_key_slots": [
"{HASHES_API_KEY}"
],
"input_type": "hash",
"output_type": [
"hash"
],
"normalization_map": {},
"tags": [
"passive"
],
"health_check_url": "https://hashes.org",
"expected_status": 200,
"reliability_score": 3,
"backup_endpoints": [],
"confidence": 0.7
}
+8
View File
@@ -139,6 +139,14 @@ SERVICE_REGISTRY: Dict[str, Dict] = {
"WHOXY_API_KEY": {"display": "Whoxy WHOIS", "public": False},
"ZEROBOUNCE_API_KEY": {"display": "ZeroBounce", "public": False},
"ZOOMEYE_API_KEY": {"display": "ZoomEye", "public": False},
# ── Added in v1.0.1 ───────────────────────────────────────────────
"EMAILREP_API_KEY": {"display": "EmailRep.io", "public": False},
"HASHES_COM_API_KEY": {"display": "Hashes.com (crack API)", "public": False},
"THREATFOX_API_KEY": {"display": "ThreatFox (abuse.ch)", "public": False},
"URLHAUS_API_KEY": {"display": "URLhaus (abuse.ch)", "public": False},
"MALWAREBAZAAR_API_KEY": {"display": "MalwareBazaar (abuse.ch)", "public": False},
"FULLHUNT_API_KEY": {"display": "FullHunt (attack surface)", "public": False},
"NETLAS_API_KEY": {"display": "Netlas.io (internet scanner)", "public": False},
}
_PRIVATE_KEYS = {k: v for k, v in SERVICE_REGISTRY.items() if not v["public"]}
+54 -24
View File
@@ -67,43 +67,47 @@ async def _query_api(session, url: str, fmt: str) -> Optional[str]:
async def async_crack(session, hash_value: str, hash_type: str) -> Optional[str]:
"""
Query multiple rainbow-table APIs concurrently.
Returns first plaintext found, or None. bcrypt is skipped.
Attempt to recover the plaintext for a given hash.
C1: create tasks upfront for cancellation, but await each via asyncio.shield
inside as_completed no double wait_for wrapping.
C2: for 32-char hex (md5/ntlm ambiguity), also query NTLM-specific APIs.
Strategy:
1. Local rockyou wordlist (no external calls, no rate limits).
2. hashes.com API if HASHES_COM_API_KEY is configured.
Per-API timeout: 8s. Global budget: 20s (CRACK_TIMEOUT).
All tasks are cancelled as soon as the first result is found.
bcrypt is skipped computationally infeasible for online cracking.
"""
if hash_type == "bcrypt":
return None
h = hash_value.strip().lower()
apis = [
(f"https://www.nitrxgen.net/md5db/{h}", "text"),
(f"https://hashes.com/en/api/hash?hash={h}", "json"),
(f"https://hash.help/api/lookup/{h}", "json"),
(f"https://hashkiller.io/api/search.php?hash={h}", "json"),
(f"https://md5decrypt.net/Api/api.php?hash={h}&hash_type={hash_type}&email=&code=", "text"),
(f"https://www.cmd5.org/api.ashx?hash={h}", "text"),
]
# C2: for 32-char hashes (md5/ntlm ambiguous), add NTLM-specific endpoint
if hash_type == "md5" and len(h) == 32:
apis.append((f"https://hashes.com/en/api/hash?hash={h}&type=ntlm", "json"))
# C1: create tasks so we can cancel them; shield each before passing to wait_for
# so cancellation of the shield future does not cancel the underlying task prematurely.
# 1. Local wordlist first — fast, zero external exposure
import concurrent.futures as _cf
loop = asyncio.get_running_loop()
with _cf.ThreadPoolExecutor(max_workers=1) as _ex:
local = await loop.run_in_executor(_ex, _local_crack_sync_blocking, hash_value, hash_type)
if local:
return local
# 2. hashes.com if API key is configured
apis = []
try:
from sources.helpers.config_handler import ConfigManager # type: ignore
hashes_com_key = ConfigManager.get_key("HASHES_COM_API_KEY")
if hashes_com_key:
apis.append((f"https://hashes.com/en/api/search?hash={h}&key={hashes_com_key}", "json"))
except Exception:
pass
if not apis:
return None
tasks = [asyncio.create_task(_query_api(session, url, fmt)) for url, fmt in apis]
result: Optional[str] = None
try:
for fut in asyncio.as_completed(tasks):
try:
res = await asyncio.wait_for(asyncio.shield(fut), timeout=_API_TIMEOUT)
except (asyncio.TimeoutError, asyncio.CancelledError):
continue
except Exception:
except (asyncio.TimeoutError, asyncio.CancelledError, Exception):
continue
if res:
result = res
@@ -111,9 +115,35 @@ async def async_crack(session, hash_value: str, hash_type: str) -> Optional[str]
except Exception:
pass
finally:
# Cancel all remaining tasks and await to suppress pending-task warnings
for t in tasks:
if not t.done():
t.cancel()
await asyncio.gather(*[t for t in tasks if not t.done()], return_exceptions=True)
return result
def _local_crack_sync_blocking(hash_value: str, hash_type: str) -> Optional[str]:
"""Pure-sync version for ThreadPoolExecutor."""
import hashlib as _hl
from pathlib import Path as _Path
wordlist = _Path.home() / ".nox" / "wordlists" / "rockyou.txt"
if not wordlist.exists():
return None
h = hash_value.strip().lower()
_hashers = {
"md5": lambda w: _hl.md5(w).hexdigest(),
"sha1": lambda w: _hl.sha1(w).hexdigest(),
"sha256": lambda w: _hl.sha256(w).hexdigest(),
}
hasher = _hashers.get(hash_type)
if not hasher:
return None
try:
with wordlist.open("rb") as f:
for line in f:
word = line.rstrip(b"\n\r")
if hasher(word) == h:
return word.decode("utf-8", errors="replace")
except Exception:
pass
return None
+4 -3
View File
@@ -1,7 +1,7 @@
{
"name": "hudsonrock_osint",
"category": "breach_data",
"endpoint": "https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-login?username={target}",
"endpoint": "https://cavalier.hudsonrock.com/api/json/v2/osint-tools/search-by-email?email={target}",
"method": "GET",
"requires_auth": false,
"selectors": {
@@ -10,10 +10,11 @@
"rate_limit": 1.0,
"headers": {},
"api_key_slots": [],
"input_type": "username",
"input_type": "email",
"output_type": [
"email",
"domain"
"domain",
"username"
],
"normalization_map": {
"stealers": "breach_record"
+33
View File
@@ -0,0 +1,33 @@
{
"name": "ipapi_is",
"category": "geolocation",
"endpoint": "https://api.ipapi.is/?q={target}",
"method": "GET",
"requires_auth": false,
"selectors": {
"org": "$.org",
"asn": "$.asn.asn",
"abuse": "$.abuse.email"
},
"rate_limit": 1.0,
"headers": {},
"api_key_slots": [],
"input_type": "ip",
"output_type": [
"domain"
],
"normalization_map": {
"org": "asn_org",
"asn": "asn_number",
"abuse": "abuse_contact"
},
"tags": [
"passive",
"fast"
],
"health_check_url": "https://api.ipapi.is",
"expected_status": 200,
"reliability_score": 4,
"backup_endpoints": [],
"confidence": 0.85
}
-31
View File
@@ -1,31 +0,0 @@
{
"name": "leakstats_pw",
"category": "breaches",
"endpoint": "https://leakstats.net/api/password/{target}",
"method": "GET",
"requires_auth": true,
"selectors": {
"count": "$.count"
},
"rate_limit": 1.0,
"headers": {
"api-key": "{LEAKSTATS_API_KEY}"
},
"api_key_slots": [
"{LEAKSTATS_API_KEY}"
],
"input_type": "hash",
"output_type": [
"hash"
],
"normalization_map": {},
"tags": [
"passive"
],
"health_check_url": "https://leakstats.net",
"expected_status": 200,
"reliability_score": 3,
"is_volatile": true,
"backup_endpoints": [],
"confidence": 0.7
}
+38
View File
@@ -0,0 +1,38 @@
{
"name": "malwarebazaar",
"category": "hashes",
"endpoint": "https://mb-api.abuse.ch/api/v1/",
"method": "POST",
"requires_auth": true,
"selectors": {
"data": "$.data"
},
"rate_limit": 1.0,
"headers": {
"API-KEY": "{MALWAREBAZAAR_API_KEY}"
},
"payload_template": {
"query": "get_info",
"hash": "{target}"
},
"api_key_slots": [
"{MALWAREBAZAAR_API_KEY}"
],
"input_type": "hash",
"output_type": [
"hash"
],
"normalization_map": {
"file_name": "filename",
"tags": "tags"
},
"tags": [
"passive",
"threat"
],
"health_check_url": "https://mb-api.abuse.ch",
"expected_status": 200,
"reliability_score": 5,
"backup_endpoints": [],
"confidence": 1.0
}
+35
View File
@@ -0,0 +1,35 @@
{
"name": "netlas_search",
"category": "scanners",
"endpoint": "https://app.netlas.io/api/responses/?q={target}&source_type=include&start=0&fields=",
"method": "GET",
"requires_auth": true,
"selectors": {
"items": "$.items"
},
"rate_limit": 1.0,
"headers": {
"X-API-Key": "{NETLAS_API_KEY}"
},
"api_key_slots": [
"{NETLAS_API_KEY}"
],
"input_type": "ip",
"output_type": [
"ip",
"domain"
],
"normalization_map": {
"data.ip": "ip_address",
"data.domain": "domain"
},
"tags": [
"passive",
"infrastructure"
],
"health_check_url": "https://app.netlas.io",
"expected_status": 200,
"reliability_score": 4,
"backup_endpoints": [],
"confidence": 0.85
}
@@ -1,11 +1,11 @@
{
"name": "checkleaked",
"name": "proxynova_comb",
"category": "breaches",
"endpoint": "https://api.checkleaked.cc/check/{target}",
"endpoint": "https://api.proxynova.com/comb?query={target}",
"method": "GET",
"requires_auth": false,
"selectors": {
"found": "$.found"
"lines": "$.lines"
},
"rate_limit": 1.0,
"headers": {},
@@ -14,17 +14,17 @@
"output_type": [
"email"
],
"normalization_map": {},
"normalization_map": {
"lines": "credential_line"
},
"tags": [
"passive",
"stealth"
],
"health_check_url": "https://api.checkleaked.cc",
"health_check_url": "https://api.proxynova.com",
"expected_status": 200,
"reliability_score": 2,
"reliability_score": 3,
"is_volatile": true,
"backup_endpoints": [
"https://checkleaked.cc/api/check/{target}"
],
"confidence": 0.55
"backup_endpoints": [],
"confidence": 0.7
}
+7 -3
View File
@@ -1,7 +1,7 @@
{
"name": "scylla_sh_search",
"category": "breaches",
"endpoint": "https://scylla.sh/search?q={target}",
"endpoint": "https://scylla.so/search?q={target}",
"method": "GET",
"requires_auth": false,
"selectors": {
@@ -20,12 +20,16 @@
"passive",
"stealth"
],
"health_check_url": "https://scylla.sh",
"health_check_url": "https://scylla.so",
"expected_status": 200,
"reliability_score": 2,
"is_volatile": true,
"bypass_required": [
"cloudflare"
],
"user_agent_type": "browser",
"backup_endpoints": [
"https://scylla.sh/api/search?q={target}"
"https://scylla.so/api/search?q={target}"
],
"confidence": 0.55
}
+34
View File
@@ -0,0 +1,34 @@
{
"name": "shodan_internetdb",
"category": "scanners",
"endpoint": "https://internetdb.shodan.io/{target}",
"method": "GET",
"requires_auth": false,
"selectors": {
"hostnames": "$.hostnames",
"ports": "$.ports",
"vulns": "$.vulns"
},
"rate_limit": 1.0,
"headers": {},
"api_key_slots": [],
"input_type": "ip",
"output_type": [
"domain",
"ip"
],
"normalization_map": {
"hostnames": "domain",
"vulns": "cve"
},
"tags": [
"passive",
"fast",
"infrastructure"
],
"health_check_url": "https://internetdb.shodan.io",
"expected_status": 200,
"reliability_score": 5,
"backup_endpoints": [],
"confidence": 1.0
}
-30
View File
@@ -1,30 +0,0 @@
{
"name": "spyse_domain",
"category": "scanners",
"endpoint": "https://api.spyse.com/v1/domain/details/{target}",
"method": "GET",
"requires_auth": true,
"selectors": {
"asn": "$.data.asn"
},
"rate_limit": 1.0,
"headers": {
"Authorization": "Bearer {SPYSE_API_KEY}"
},
"api_key_slots": [
"{SPYSE_API_KEY}"
],
"input_type": "domain",
"output_type": [
"ip"
],
"normalization_map": {},
"tags": [
"passive"
],
"health_check_url": "https://api.spyse.com",
"expected_status": 200,
"reliability_score": 3,
"backup_endpoints": [],
"confidence": 0.7
}
-30
View File
@@ -1,30 +0,0 @@
{
"name": "spyse_ip",
"category": "scanners",
"endpoint": "https://api.spyse.com/v1/ip/details/{target}",
"method": "GET",
"requires_auth": true,
"selectors": {
"geo": "$.data.geo"
},
"rate_limit": 1.0,
"headers": {
"Authorization": "Bearer {SPYSE_API_KEY}"
},
"api_key_slots": [
"{SPYSE_API_KEY}"
],
"input_type": "ip",
"output_type": [
"ip"
],
"normalization_map": {},
"tags": [
"passive"
],
"health_check_url": "https://api.spyse.com",
"expected_status": 200,
"reliability_score": 3,
"backup_endpoints": [],
"confidence": 0.7
}
-32
View File
@@ -1,32 +0,0 @@
{
"name": "threatcrowd_domain",
"category": "threat_intel",
"endpoint": "https://www.threatcrowd.org/searchApi/v2/domain/report/?domain={target}",
"method": "GET",
"requires_auth": false,
"selectors": {
"ips": "$.resolutions[*].ip_address"
},
"rate_limit": 5.0,
"headers": {},
"api_key_slots": [],
"input_type": "domain",
"output_type": [
"ip"
],
"normalization_map": {},
"tags": [
"passive",
"threat"
],
"health_check_url": "https://www.threatcrowd.org",
"expected_status": 200,
"reliability_score": 3,
"is_volatile": true,
"bypass_required": [
"cloudflare"
],
"user_agent_type": "browser",
"backup_endpoints": [],
"confidence": 0.7
}
-32
View File
@@ -1,32 +0,0 @@
{
"name": "threatcrowd_email",
"category": "threat_intel",
"endpoint": "https://www.threatcrowd.org/searchApi/v2/email/report/?email={target}",
"method": "GET",
"requires_auth": false,
"selectors": {
"domains": "$.domains"
},
"rate_limit": 5.0,
"headers": {},
"api_key_slots": [],
"input_type": "email",
"output_type": [
"domain"
],
"normalization_map": {},
"tags": [
"passive",
"threat"
],
"health_check_url": "https://www.threatcrowd.org",
"expected_status": 200,
"reliability_score": 3,
"is_volatile": true,
"bypass_required": [
"cloudflare"
],
"user_agent_type": "browser",
"backup_endpoints": [],
"confidence": 0.7
}
+41
View File
@@ -0,0 +1,41 @@
{
"name": "threatfox",
"category": "threat_intel",
"endpoint": "https://threatfox-api.abuse.ch/api/v1/",
"method": "POST",
"requires_auth": true,
"selectors": {
"results": "$.data"
},
"rate_limit": 1.0,
"headers": {
"API-KEY": "{THREATFOX_API_KEY}",
"Content-Type": "application/json"
},
"payload_template": {
"query": "search_ioc",
"search_term": "{target}"
},
"api_key_slots": [
"{THREATFOX_API_KEY}"
],
"input_type": "any",
"output_type": [
"ip",
"domain",
"hash"
],
"normalization_map": {
"ioc": "indicator",
"malware": "malware_family"
},
"tags": [
"passive",
"threat"
],
"health_check_url": "https://threatfox-api.abuse.ch",
"expected_status": 200,
"reliability_score": 5,
"backup_endpoints": [],
"confidence": 1.0
}
+38
View File
@@ -0,0 +1,38 @@
{
"name": "urlhaus",
"category": "threat_intel",
"endpoint": "https://urlhaus-api.abuse.ch/v1/host/",
"method": "POST",
"requires_auth": true,
"selectors": {
"urls": "$.urls"
},
"rate_limit": 1.0,
"headers": {
"Auth-Key": "{URLHAUS_API_KEY}"
},
"payload_template": {
"host": "{target}"
},
"api_key_slots": [
"{URLHAUS_API_KEY}"
],
"input_type": "domain",
"output_type": [
"url",
"domain"
],
"normalization_map": {
"url": "malware_url",
"threat": "threat_type"
},
"tags": [
"passive",
"threat"
],
"health_check_url": "https://urlhaus-api.abuse.ch",
"expected_status": 200,
"reliability_score": 5,
"backup_endpoints": [],
"confidence": 1.0
}