From 61bd9b555592885177b0093d1db17b7a068c86ae Mon Sep 17 00:00:00 2001 From: nox-project Date: Wed, 22 Apr 2026 15:49:51 +0200 Subject: [PATCH] release: v1.0.4 --- CHANGELOG.md | 17 +++++++++++++++++ README.md | 2 +- build_deb.sh | 4 ++-- build_sources.py | 8 ++++++-- docs/nox-cli.1 | 2 +- nox.py | 18 +++++++++++++----- pyproject.toml | 2 +- setup.py | 2 +- sources/circl_hashlookup.json | 5 ++++- sources/crt_sh.json | 5 +++-- sources/helpers/config_handler.py | 3 ++- sources/helpers/reporting.py | 6 +++--- sources/helpers/scanner.py | 1 + 13 files changed, 55 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 018b0af..56a0ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ All notable changes to NOX are documented here. +## [1.0.4] — 2026-04-22 + +### Engine +- **Fixed:** `_build_ssl_context` — custom TLS context had zero CA certificates loaded, causing `SSLCertVerificationError` on all HTTPS connections. CA bundle now loaded via `certifi` with `load_default_certs()` fallback. +- **Fixed:** `_NOISE_RE` in reporting — `ssl.`, `aiohttp.`, `asyncio.` were substring-matched, silently zeroing legitimate emails and domains (e.g. `user@ssl.example.com`). Patterns now anchored to start-of-string or whitespace. +- **Fixed:** `COMBO_RE` in `ScrapeEngine` — multiline greedy match produced credentials with embedded newlines in email/password fields. Pattern now excludes newlines from both capture groups. +- **Fixed:** `_in_flight` dict in `AvalancheScanner` — entries were never removed after processing, causing unbounded memory growth on deep scans. Entry is now popped in the `finally` block after the future resolves. +- **Fixed:** `DorkingEngine.__init__` — `ProxyManager.get_proxies()` was called eagerly on every `Orchestrator` instantiation, triggering a proxy fetch and OPSEC warning even for local-only commands (`--crack`, `--list-sources`, `--analyze`). Proxy fetch is now lazy. +- **Fixed:** `ScrapeEngine._fetch_content` — IntelX paste content fetch used `DB.get_key("intelx")` which does not read `apikeys.json`. Replaced with `Vault.get("INTELX_API_KEY")` for consistent key resolution. +- **Fixed:** `AsyncSource._rec` and `RiskEngine.score` — plugin `confidence` values were stored in `NoxSourceProvider._confidence` but never transferred to `Record.source_confidence`. All 124 plugin records received a flat `0.5` confidence regardless of their declared value. `_rec()` now injects `self._confidence` into the record; `RiskEngine.score()` uses `record.source_confidence` as fallback when the source is not in `_SRC_CONFIDENCE`. +- **Fixed:** `ConfigManager.get_key` — `None` results were cached, preventing env vars set after the first lookup from being detected in the same session. Only positive values are now cached. +- **Fixed:** Recursive Avalanche Engine — identifiers extracted from paste content (`paste["patterns"]`) were not being harvested as pivot seeds. All `scrape_res["pastes"]` pattern matches are now fed into `_extract_ids_from_text`. + +### Sources +- **Fixed:** `circl_hashlookup` — endpoint hardcoded to `/lookup/md5/{target}` but `input_type=hash` accepts SHA1/SHA256. SHA1 and SHA256 backup endpoints added; engine now routes each hash type to the correct path. +- **Updated:** `crt_sh` — `reliability_score` lowered from 5 to 3, `is_volatile=true` added to reflect documented intermittent availability. + ## [1.0.3] — 2026-04-15 ### Engine diff --git a/README.md b/README.md index df34969..eccebdc 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Cyber Threat Intelligence Framework** -[![Status](https://img.shields.io/badge/Status-v1.0.3-success)](https://github.com/nox-project/nox-framework/releases/tag/v1.0.3) +[![Status](https://img.shields.io/badge/Status-v1.0.4-success)](https://github.com/nox-project/nox-framework/releases/tag/v1.0.4) [![Python](https://img.shields.io/badge/Python-3.8%2B-blue?logo=python&logoColor=white)](https://www.python.org/) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE.txt) [![Kali Linux](https://img.shields.io/badge/Kali%20Linux-Ready-557C94?logo=kalilinux&logoColor=white)](https://www.kali.org/) diff --git a/build_deb.sh b/build_deb.sh index fb866d1..a08227c 100755 --- a/build_deb.sh +++ b/build_deb.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -e -# NOX v1.0.3 — .deb build script (FPM) +# NOX v1.0.4 — .deb build script (FPM) # Requires: fpm → gem install fpm -VERSION="1.0.3" +VERSION="1.0.4" PKG_NAME="nox-cli" ARCH="all" OUT_DIR="dist" diff --git a/build_sources.py b/build_sources.py index f6c8398..6c6933b 100644 --- a/build_sources.py +++ b/build_sources.py @@ -188,7 +188,7 @@ FREE_PUBLIC_SOURCES: List[SourceConfig] = [ input_type="domain", output_type=["domain"], normalization_map={"name_value": "domain"}, tags=["passive", "fast"], - health_check_url="https://crt.sh", reliability_score=5), + health_check_url="https://crt.sh", reliability_score=3, is_volatile=True), _base("hackertarget_hostsearch", "dns_recon", "https://api.hackertarget.com/hostsearch/?q={target}", "GET", @@ -521,7 +521,11 @@ FREE_PUBLIC_SOURCES: List[SourceConfig] = [ normalization_map={"FileName": "filename", "MD5": "hash_md5"}, tags=["passive", "fast"], health_check_url="https://hashlookup.circl.lu", - reliability_score=5), + reliability_score=5, + backup_endpoints=[ + "https://hashlookup.circl.lu/lookup/sha1/{target}", + "https://hashlookup.circl.lu/lookup/sha256/{target}", + ]), _base("ipapi_is", "geolocation", "https://api.ipapi.is/?q={target}", "GET", diff --git a/docs/nox-cli.1 b/docs/nox-cli.1 index b28f3ae..c878981 100644 --- a/docs/nox-cli.1 +++ b/docs/nox-cli.1 @@ -1,4 +1,4 @@ -.TH NOX\-CLI 1 "2026-03-30" "1.0.0" "NOX Framework" +.TH NOX\-CLI 1 "2026-04-16" "1.0.4" "NOX Framework" .SH NAME nox-cli \- Advanced Asynchronous Cyber Threat Intelligence Framework .SH SYNOPSIS diff --git a/nox.py b/nox.py index 76eaca3..e885b78 100644 --- a/nox.py +++ b/nox.py @@ -150,7 +150,7 @@ except Exception: VERSION = _sp2.check_output(["dpkg-query", "-W", "-f=${Version}", "nox-cli"], stderr=_sp2.DEVNULL).decode().strip() or VERSION except Exception: pass -BUILD_DATE = "2026-04-15" +BUILD_DATE = "2026-04-16" # ── Smart Path Layout ────────────────────────────────────────────────── HOME_NOX = Path.home() / ".nox" @@ -646,7 +646,7 @@ class RiskEngine: @staticmethod def score(record: "Record") -> "Record": - conf = _SRC_CONFIDENCE.get(record.source, 0.5) + conf = _SRC_CONFIDENCE.get(record.source, record.source_confidence) record.source_confidence = conf dtypes_str = " ".join(record.data_types).lower() if record.data_types else "" @@ -1540,6 +1540,11 @@ def _build_ssl_context() -> ssl.SSLContext: ctx.set_ciphers(Cfg.TLS_CIPHERS) ctx.check_hostname = True ctx.verify_mode = ssl.CERT_REQUIRED + try: + import certifi + ctx.load_verify_locations(certifi.where()) + except Exception: + ctx.load_default_certs() return ctx @@ -1702,6 +1707,7 @@ class AsyncSource(ABC): def _rec(self, **kw) -> Record: kw.setdefault("source", self.name) + kw.setdefault("source_confidence", getattr(self, "_confidence", 0.5)) sev = kw.pop("severity", Severity.MEDIUM) r = Record(**{k: v for k, v in kw.items() if k in Record.__dataclass_fields__}) r.severity = sev @@ -2236,10 +2242,12 @@ class DorkingEngine(Src): super().__init__(*args, **kwargs) self._dead_proxies: set = set() self._proxy_index: int = 0 - self.proxies = ProxyManager.get_proxies() + self.proxies: list = [] self._dead_instances: set = set() def _get_next_proxy(self) -> Optional[str]: + if not self.proxies: + self.proxies = ProxyManager.get_proxies() live = [p for p in self.proxies if p not in self._dead_proxies] if not live: return None @@ -2498,7 +2506,7 @@ class ScrapeEngine: CRED_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.-]+\s*[:;|]\s*\S+", re.IGNORECASE) EMAIL_RE = re.compile(r"[\w.+-]+@[\w-]+\.[\w.]+") HASH_RE = re.compile(r"\b[a-f0-9]{32,128}\b", re.IGNORECASE) - COMBO_RE = re.compile(r"^[^:]+:[^:]+$", re.MULTILINE) + COMBO_RE = re.compile(r"^[^\n:]+:[^\n:]+$", re.MULTILINE) PATTERNS = [ (re.compile(r"(?:password|passwd|pass|pwd)\s*[:=]\s*\S+", re.I), "Password"), @@ -2722,7 +2730,7 @@ class ScrapeEngine: return "" raw_urls: dict = {} # paste fetch URLs — resolved per site name if site == "IntelX": - key = self.db.get_key("intelx") + key = Vault.get("INTELX_API_KEY") if key: resp = self.s.get(f"https://2.intelx.io/file/read?type=1&systemid={pid}&k={key}", timeout=15) if resp.ok: diff --git a/pyproject.toml b/pyproject.toml index 19f591e..aa25025 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nox-cli" -version = "1.0.3" +version = "1.0.4" description = "Advanced Asynchronous Cyber Threat Intelligence Framework" readme = { file = "README.md", content-type = "text/markdown" } license = { text = "Apache-2.0" } diff --git a/setup.py b/setup.py index ef86188..5b64115 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ requirements = [ setup( name="nox-cli", - version="1.0.0", + version="1.0.4", author="nox-project", description="Advanced Asynchronous Cyber Threat Intelligence Framework", long_description=Path("README.md").read_text(), diff --git a/sources/circl_hashlookup.json b/sources/circl_hashlookup.json index 183e192..7b6c642 100644 --- a/sources/circl_hashlookup.json +++ b/sources/circl_hashlookup.json @@ -26,6 +26,9 @@ "health_check_url": "https://hashlookup.circl.lu", "expected_status": 200, "reliability_score": 5, - "backup_endpoints": [], + "backup_endpoints": [ + "https://hashlookup.circl.lu/lookup/sha1/{target}", + "https://hashlookup.circl.lu/lookup/sha256/{target}" + ], "confidence": 1.0 } \ No newline at end of file diff --git a/sources/crt_sh.json b/sources/crt_sh.json index e4e64e8..38dce37 100644 --- a/sources/crt_sh.json +++ b/sources/crt_sh.json @@ -25,7 +25,8 @@ ], "health_check_url": "https://crt.sh", "expected_status": 200, - "reliability_score": 5, + "reliability_score": 3, + "is_volatile": true, "backup_endpoints": [], - "confidence": 1.0 + "confidence": 0.7 } \ No newline at end of file diff --git a/sources/helpers/config_handler.py b/sources/helpers/config_handler.py index ce04737..30b7bbe 100644 --- a/sources/helpers/config_handler.py +++ b/sources/helpers/config_handler.py @@ -228,7 +228,8 @@ class ConfigManager: return cls._cache[key_name] val = os.environ.get(key_name, "") or cls._get_store().get(key_name, "") result = None if (not val or val == UNIVERSAL_PLACEHOLDER) else val - cls._cache[key_name] = result + if result is not None: + cls._cache[key_name] = result return result # Backward-compatible alias used by nox.py internals diff --git a/sources/helpers/reporting.py b/sources/helpers/reporting.py index 7e6c9cc..bcb0db4 100644 --- a/sources/helpers/reporting.py +++ b/sources/helpers/reporting.py @@ -14,9 +14,9 @@ from typing import Any, Dict, List # ── Noise patterns stripped from all report output ──────────────────── _NOISE_RE = re.compile( r"(Traceback \(most recent|File \".*\.py\"|TimeoutError|ProxyError" - r"|ConnectionError|aiohttp\.|ClientConnector|ssl\.|asyncio\." - r"|Task exception|NoneType|Object of type)", - re.I, + r"|ConnectionError|ClientConnector|Task exception|NoneType|Object of type" + r"|(?:^|[\s(])aiohttp\.|(?:^|[\s(])asyncio\.|(?:^|[\s(])ssl\.)", + re.I | re.MULTILINE, ) _CTRL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]") diff --git a/sources/helpers/scanner.py b/sources/helpers/scanner.py index e71f856..80593ac 100644 --- a/sources/helpers/scanner.py +++ b/sources/helpers/scanner.py @@ -227,6 +227,7 @@ class AvalancheScanner: finally: if not fut.done(): fut.set_result(None) + self._in_flight.pop(key, None) # ── Core pipeline ─────────────────────────────────────────────────