diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a0ccb..04a3157 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ All notable changes to NOX are documented here. +## [1.0.5] — 2026-05-06 + +### Sources +- **Fixed:** `pypi_user` — endpoint was querying the PyPI package API (`/pypi/{target}/json`) instead of user profiles, returning package metadata for any username that matched a package name and 404 for all others. Source now uses the PyPI XML-RPC `user_packages` method, returning the actual list of packages owned by the target username. + +### Engine +- **Fixed:** `Detect.qtype` — dotted-quad strings with out-of-range octets (e.g. `255.255.255.256`) were classified as `phone` instead of `username`. Invalid IPs now fall through to `username`. +- **Fixed:** `Record.dedup_key` — records with no identity fields (email, username, password) from different sources all produced the same SHA-256 hash, causing all but the first to be silently dropped from the DB. Source name is now included in the key when identity fields are absent. +- **Added:** `raw_payload` support in `NoxSourceProvider` — POST sources can now send raw string bodies (e.g. XML-RPC) in addition to JSON and form-encoded payloads. +- **Fixed:** `FileSystemProvider._by_regex` — regex-extracted values that are not email addresses (e.g. package names, usernames) are now correctly placed in the `username` field instead of `email`. + ## [1.0.4] — 2026-04-22 ### Engine diff --git a/README.md b/README.md index eccebdc..c05f9ea 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ **Cyber Threat Intelligence Framework** -[![Status](https://img.shields.io/badge/Status-v1.0.4-success)](https://github.com/nox-project/nox-framework/releases/tag/v1.0.4) +[![Status](https://img.shields.io/badge/Status-v1.0.5-success)](https://github.com/nox-project/nox-framework/releases/tag/v1.0.5) [![Python](https://img.shields.io/badge/Python-3.8%2B-blue?logo=python&logoColor=white)](https://www.python.org/) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE.txt) [![Kali Linux](https://img.shields.io/badge/Kali%20Linux-Ready-557C94?logo=kalilinux&logoColor=white)](https://www.kali.org/) diff --git a/build_deb.sh b/build_deb.sh index a08227c..eef0876 100755 --- a/build_deb.sh +++ b/build_deb.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash set -e -# NOX v1.0.4 — .deb build script (FPM) +# NOX v1.0.5 — .deb build script (FPM) # Requires: fpm → gem install fpm -VERSION="1.0.4" +VERSION="1.0.5" PKG_NAME="nox-cli" ARCH="all" OUT_DIR="dist" diff --git a/build_sources.py b/build_sources.py index 6c6933b..8c444ad 100644 --- a/build_sources.py +++ b/build_sources.py @@ -53,6 +53,8 @@ class SourceConfig(BaseModel): rate_limit: float = 1.0 headers: Dict[str, str] = Field(default_factory=dict) payload_template: Optional[Dict[str, Any]] = None + raw_payload: Optional[str] = None + regex_pattern: Optional[str] = None api_key_slots: List[str] = Field(default_factory=list) # ── Typing & pivoting ─────────────────────────────────────────────────── @@ -121,6 +123,8 @@ def _mk( rate_limit: float = 1.0, headers: Optional[Dict[str, str]] = None, payload_template: Optional[Dict[str, Any]] = None, + raw_payload: Optional[str] = None, + regex_pattern: Optional[str] = None, api_key_slots: Optional[List[str]] = None, input_type: InputType = "any", output_type: Optional[List[str]] = None, @@ -146,6 +150,8 @@ def _mk( rate_limit=rate_limit, headers=headers or {}, payload_template=payload_template, + raw_payload=raw_payload, + regex_pattern=regex_pattern, api_key_slots=api_key_slots or [], input_type=input_type, output_type=output_type or [], @@ -416,11 +422,15 @@ FREE_PUBLIC_SOURCES: List[SourceConfig] = [ health_check_url="https://ipvigilante.com", reliability_score=3, is_volatile=True), _base("pypi_user", "social", - "https://pypi.org/pypi/{target}/json", "GET", - {"info": "$.info"}, + "https://pypi.org/pypi", "POST", + {}, + headers={"Content-Type": "text/xml"}, input_type="username", output_type=["username"], + normalization_map={}, + regex_pattern=r"Owner\s*([^<]+)", tags=["passive"], - health_check_url="https://pypi.org", reliability_score=5), + health_check_url="https://pypi.org", reliability_score=4, + raw_payload="user_packages{target}"), _base("npm_user", "social", "https://registry.npmjs.org/-/v1/search?text=maintainer:{target}", "GET", diff --git a/docs/nox-cli.1 b/docs/nox-cli.1 index c878981..f8227ea 100644 --- a/docs/nox-cli.1 +++ b/docs/nox-cli.1 @@ -1,4 +1,4 @@ -.TH NOX\-CLI 1 "2026-04-16" "1.0.4" "NOX Framework" +.TH NOX\-CLI 1 "2026-05-06" "1.0.5" "NOX Framework" .SH NAME nox-cli \- Advanced Asynchronous Cyber Threat Intelligence Framework .SH SYNOPSIS diff --git a/nox.py b/nox.py index 628e927..b2f7438 100644 --- a/nox.py +++ b/nox.py @@ -150,7 +150,7 @@ except Exception: VERSION = _sp2.check_output(["dpkg-query", "-W", "-f=${Version}", "nox-cli"], stderr=_sp2.DEVNULL).decode().strip() or VERSION except Exception: pass -BUILD_DATE = "2026-04-16" +BUILD_DATE = "2026-05-06" # ── Smart Path Layout ────────────────────────────────────────────────── HOME_NOX = Path.home() / ".nox" @@ -618,9 +618,13 @@ class Record: return d def dedup_key(self) -> str: - """SHA-256 of normalised email:password for cross-source deduplication.""" + """SHA-256 of normalised email:password for cross-source deduplication. + When identity fields are absent, source is included to prevent enrichment + records from different sources colliding into a single DB entry.""" em = (self.email or self.username or "").lower().strip() pw = (self.password or self.password_hash or "").strip() + if not em and not pw: + return hashlib.sha256(f"{self.source}:{em}:{pw}".encode()).hexdigest() return hashlib.sha256(f"{em}:{pw}".encode()).hexdigest() def get_fingerprint(self) -> str: @@ -1738,7 +1742,7 @@ class AsyncSource(ABC): _syslog.debug("API_FAIL source=%s url=%s error=%s", self.name, url[:80], exc) return 0, "", b"" - async def _post(self, session: "aiohttp.ClientSession", url: str, json_data: Dict = None, data: Dict = None, headers: Dict = None, timeout: int = None) -> Tuple[int, str, bytes]: + async def _post(self, session: "aiohttp.ClientSession", url: str, json_data: Dict = None, data: Dict = None, headers: Dict = None, timeout: int = None, raw_body: str = None) -> Tuple[int, str, bytes]: """Perform a POST with jitter and retry logic.""" await _jitter(self._config) to = aiohttp_mod.ClientTimeout(total=timeout or self._config.timeout) if aiohttp_mod else None @@ -1746,7 +1750,18 @@ class AsyncSource(ABC): for attempt in range(Cfg.RETRIES): try: async with self._sem: - if json_data is not None: + if raw_body is not None: + async with session.post(url, data=raw_body.encode(), headers=hdrs, timeout=to, ssl=_SSL_CTX) as resp: + if resp.status == 429: + retry_after = _parse_retry_after(resp.headers.get("Retry-After", ""), Cfg.RETRY_DELAY * (attempt + 2)) + _syslog.info("RATE_LIMIT source=%s url=%s retry_after=%ds", self.name, url[:80], retry_after) + await asyncio.sleep(min(retry_after, 30)) + continue + body = await resp.read() + if resp.status >= 400: + _syslog.warning("API_ERROR source=%s status=%d url=%s", self.name, resp.status, url[:80]) + return resp.status, await resp.text(errors="replace"), body + elif json_data is not None: hdrs["Content-Type"] = "application/json" async with session.post(url, json=json_data, headers=hdrs, timeout=to, ssl=_SSL_CTX) as resp: if resp.status == 429: @@ -1840,6 +1855,7 @@ class Detect: q = q.strip() if re.match(r"^[\w.+-]+@[\w-]+\.[\w.]+$", q): return "email" if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", q) and all(0 <= int(o) <= 255 for o in q.split(".")): return "ip" + if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", q): return "username" if re.match(r"^(\+?\d{1,3}[\s.-]?)?\(?\d{2,4}\)?[\s.-]?\d{3,4}[\s.-]?\d{3,4}$", q): return "phone" if re.match(r"^[a-fA-F0-9]{32,128}$", q): return "hash" if re.match(r"^\$2[aby]?\$", q) or re.match(r"^\$argon2", q) or re.match(r"^\$[156]\$", q): return "hash" @@ -6274,8 +6290,12 @@ class FileSystemProvider(AsyncSource): records = [] for m in re.finditer(pattern, text): groups = m.groups() + val = groups[0] if len(groups) > 0 else "" + # Route to username if the value is not an email address + is_email = "@" in val records.append(self._rec( - email = groups[0] if len(groups) > 0 else "", + email = val if is_email else "", + username = val if not is_email else "", password = groups[1] if len(groups) > 1 else "", breach_name = self.name, data_types = [self.name, "Credentials"], @@ -6406,9 +6426,14 @@ class NoxSourceProvider(FileSystemProvider): payload = _sub(d.get("payload") or {}) if method == "POST": - status, text, _ = await self._post(session, url, - json_data=payload or None, - headers=hdrs) + if isinstance(payload, str): + status, text, _ = await self._post(session, url, + raw_body=payload, + headers=hdrs) + else: + status, text, _ = await self._post(session, url, + json_data=payload or None, + headers=hdrs) else: status, text, _ = await self._get(session, url, headers=hdrs) @@ -6575,8 +6600,8 @@ class SourceOrchestrator: "output_type": raw.get("output_type", []), "pivot_types": raw.get("pivot_types", []), "confidence": raw.get("confidence", 0.5), - # payload_template → payload for POST sources - "payload": raw.get("payload_template") or raw.get("payload") or {}, + # payload_template → payload for POST sources; raw_payload takes precedence + "payload": raw.get("raw_payload") or raw.get("payload_template") or raw.get("payload") or {}, # Pass resolved slot keys so FileSystemProvider can use them "_slot_keys": slot_keys, # Two-phase poll support diff --git a/pyproject.toml b/pyproject.toml index aa25025..880203e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nox-cli" -version = "1.0.4" +version = "1.0.5" description = "Advanced Asynchronous Cyber Threat Intelligence Framework" readme = { file = "README.md", content-type = "text/markdown" } license = { text = "Apache-2.0" } diff --git a/setup.py b/setup.py index 5b64115..92a3f87 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ requirements = [ setup( name="nox-cli", - version="1.0.4", + version="1.0.5", author="nox-project", description="Advanced Asynchronous Cyber Threat Intelligence Framework", long_description=Path("README.md").read_text(), diff --git a/sources/pypi_user.json b/sources/pypi_user.json index a080111..e25c901 100644 --- a/sources/pypi_user.json +++ b/sources/pypi_user.json @@ -1,14 +1,16 @@ { "name": "pypi_user", "category": "social", - "endpoint": "https://pypi.org/pypi/{target}/json", - "method": "GET", + "endpoint": "https://pypi.org/pypi", + "method": "POST", "requires_auth": false, - "selectors": { - "info": "$.info" - }, + "selectors": {}, "rate_limit": 1.0, - "headers": {}, + "headers": { + "Content-Type": "text/xml" + }, + "raw_payload": "user_packages{target}", + "regex_pattern": "Owner\\s*([^<]+)", "api_key_slots": [], "input_type": "username", "output_type": [ @@ -20,7 +22,7 @@ ], "health_check_url": "https://pypi.org", "expected_status": 200, - "reliability_score": 5, + "reliability_score": 4, "backup_endpoints": [], - "confidence": 1.0 + "confidence": 0.85 } \ No newline at end of file