release: v1.0.5

2026-06-08 08:05:50 +00:00 · 2026-05-06 14:49:45 +02:00
parent edcbf99fa7
commit 0814bfed2e
9 changed files with 75 additions and 27 deletions
@@ -2,6 +2,17 @@

 All notable changes to NOX are documented here.

+## [1.0.5] — 2026-05-06
+
+### Sources
+- **Fixed:** `pypi_user` — endpoint was querying the PyPI package API (`/pypi/{target}/json`) instead of user profiles, returning package metadata for any username that matched a package name and 404 for all others. Source now uses the PyPI XML-RPC `user_packages` method, returning the actual list of packages owned by the target username.
+
+### Engine
+- **Fixed:** `Detect.qtype` — dotted-quad strings with out-of-range octets (e.g. `255.255.255.256`) were classified as `phone` instead of `username`. Invalid IPs now fall through to `username`.
+- **Fixed:** `Record.dedup_key` — records with no identity fields (email, username, password) from different sources all produced the same SHA-256 hash, causing all but the first to be silently dropped from the DB. Source name is now included in the key when identity fields are absent.
+- **Added:** `raw_payload` support in `NoxSourceProvider` — POST sources can now send raw string bodies (e.g. XML-RPC) in addition to JSON and form-encoded payloads.
+- **Fixed:** `FileSystemProvider._by_regex` — regex-extracted values that are not email addresses (e.g. package names, usernames) are now correctly placed in the `username` field instead of `email`.
+
 ## [1.0.4] — 2026-04-22

 ### Engine
@@ -11,7 +11,7 @@

 **Cyber Threat Intelligence Framework**

-[![Status](https://img.shields.io/badge/Status-v1.0.4-success)](https://github.com/nox-project/nox-framework/releases/tag/v1.0.4)
+[![Status](https://img.shields.io/badge/Status-v1.0.5-success)](https://github.com/nox-project/nox-framework/releases/tag/v1.0.5)
 [![Python](https://img.shields.io/badge/Python-3.8%2B-blue?logo=python&logoColor=white)](https://www.python.org/)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE.txt)
 [![Kali Linux](https://img.shields.io/badge/Kali%20Linux-Ready-557C94?logo=kalilinux&logoColor=white)](https://www.kali.org/)
@@ -1,10 +1,10 @@
 #!/usr/bin/env bash
 set -e

-# NOX v1.0.4 — .deb build script (FPM)
+# NOX v1.0.5 — .deb build script (FPM)
 # Requires: fpm  →  gem install fpm

-VERSION="1.0.4"
+VERSION="1.0.5"
 PKG_NAME="nox-cli"
 ARCH="all"
 OUT_DIR="dist"
@@ -53,6 +53,8 @@ class SourceConfig(BaseModel):
    rate_limit:       float                    = 1.0
    headers:          Dict[str, str]           = Field(default_factory=dict)
    payload_template: Optional[Dict[str, Any]] = None
+    raw_payload:      Optional[str]            = None
+    regex_pattern:    Optional[str]            = None
    api_key_slots:    List[str]                = Field(default_factory=list)

    # ── Typing & pivoting ───────────────────────────────────────────────────
@@ -121,6 +123,8 @@ def _mk(
    rate_limit: float = 1.0,
    headers: Optional[Dict[str, str]] = None,
    payload_template: Optional[Dict[str, Any]] = None,
+    raw_payload: Optional[str] = None,
+    regex_pattern: Optional[str] = None,
    api_key_slots: Optional[List[str]] = None,
    input_type: InputType = "any",
    output_type: Optional[List[str]] = None,
@@ -146,6 +150,8 @@ def _mk(
        rate_limit=rate_limit,
        headers=headers or {},
        payload_template=payload_template,
+        raw_payload=raw_payload,
+        regex_pattern=regex_pattern,
        api_key_slots=api_key_slots or [],
        input_type=input_type,
        output_type=output_type or [],
@@ -416,11 +422,15 @@ FREE_PUBLIC_SOURCES: List[SourceConfig] = [
          health_check_url="https://ipvigilante.com", reliability_score=3, is_volatile=True),

    _base("pypi_user", "social",
-          "https://pypi.org/pypi/{target}/json", "GET",
-          {"info": "$.info"},
+          "https://pypi.org/pypi", "POST",
+          {},
+          headers={"Content-Type": "text/xml"},
          input_type="username", output_type=["username"],
+          normalization_map={},
+          regex_pattern=r"<string>Owner</string></value>\s*<value><string>([^<]+)</string>",
          tags=["passive"],
-          health_check_url="https://pypi.org", reliability_score=5),
+          health_check_url="https://pypi.org", reliability_score=4,
+          raw_payload="<?xml version='1.0'?><methodCall><methodName>user_packages</methodName><params><param><value>{target}</value></param></params></methodCall>"),

    _base("npm_user", "social",
          "https://registry.npmjs.org/-/v1/search?text=maintainer:{target}", "GET",
@@ -1,4 +1,4 @@
-.TH NOX\-CLI 1 "2026-04-16" "1.0.4" "NOX Framework"
+.TH NOX\-CLI 1 "2026-05-06" "1.0.5" "NOX Framework"
 .SH NAME
 nox-cli \- Advanced Asynchronous Cyber Threat Intelligence Framework
 .SH SYNOPSIS
@@ -150,7 +150,7 @@ except Exception:
            VERSION = _sp2.check_output(["dpkg-query", "-W", "-f=${Version}", "nox-cli"], stderr=_sp2.DEVNULL).decode().strip() or VERSION
        except Exception:
            pass
-BUILD_DATE = "2026-04-16"
+BUILD_DATE = "2026-05-06"

 # ── Smart Path Layout ──────────────────────────────────────────────────
 HOME_NOX    = Path.home() / ".nox"
@@ -618,9 +618,13 @@ class Record:
        return d

    def dedup_key(self) -> str:
-        """SHA-256 of normalised email:password for cross-source deduplication."""
+        """SHA-256 of normalised email:password for cross-source deduplication.
+        When identity fields are absent, source is included to prevent enrichment
+        records from different sources colliding into a single DB entry."""
        em = (self.email or self.username or "").lower().strip()
        pw = (self.password or self.password_hash or "").strip()
+        if not em and not pw:
+            return hashlib.sha256(f"{self.source}:{em}:{pw}".encode()).hexdigest()
        return hashlib.sha256(f"{em}:{pw}".encode()).hexdigest()

    def get_fingerprint(self) -> str:
@@ -1738,7 +1742,7 @@ class AsyncSource(ABC):
                _syslog.debug("API_FAIL source=%s url=%s error=%s", self.name, url[:80], exc)
        return 0, "", b""

-    async def _post(self, session: "aiohttp.ClientSession", url: str, json_data: Dict = None, data: Dict = None, headers: Dict = None, timeout: int = None) -> Tuple[int, str, bytes]:
+    async def _post(self, session: "aiohttp.ClientSession", url: str, json_data: Dict = None, data: Dict = None, headers: Dict = None, timeout: int = None, raw_body: str = None) -> Tuple[int, str, bytes]:
        """Perform a POST with jitter and retry logic."""
        await _jitter(self._config)
        to   = aiohttp_mod.ClientTimeout(total=timeout or self._config.timeout) if aiohttp_mod else None
@@ -1746,7 +1750,18 @@ class AsyncSource(ABC):
        for attempt in range(Cfg.RETRIES):
            try:
                async with self._sem:
-                    if json_data is not None:
+                    if raw_body is not None:
+                        async with session.post(url, data=raw_body.encode(), headers=hdrs, timeout=to, ssl=_SSL_CTX) as resp:
+                            if resp.status == 429:
+                                retry_after = _parse_retry_after(resp.headers.get("Retry-After", ""), Cfg.RETRY_DELAY * (attempt + 2))
+                                _syslog.info("RATE_LIMIT source=%s url=%s retry_after=%ds", self.name, url[:80], retry_after)
+                                await asyncio.sleep(min(retry_after, 30))
+                                continue
+                            body = await resp.read()
+                            if resp.status >= 400:
+                                _syslog.warning("API_ERROR source=%s status=%d url=%s", self.name, resp.status, url[:80])
+                            return resp.status, await resp.text(errors="replace"), body
+                    elif json_data is not None:
                        hdrs["Content-Type"] = "application/json"
                        async with session.post(url, json=json_data, headers=hdrs, timeout=to, ssl=_SSL_CTX) as resp:
                            if resp.status == 429:
@@ -1840,6 +1855,7 @@ class Detect:
        q = q.strip()
        if re.match(r"^[\w.+-]+@[\w-]+\.[\w.]+$", q):                                                    return "email"
        if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", q) and all(0 <= int(o) <= 255 for o in q.split(".")): return "ip"
+        if re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", q):                                                    return "username"
        if re.match(r"^(\+?\d{1,3}[\s.-]?)?\(?\d{2,4}\)?[\s.-]?\d{3,4}[\s.-]?\d{3,4}$", q):             return "phone"
        if re.match(r"^[a-fA-F0-9]{32,128}$", q):                                                        return "hash"
        if re.match(r"^\$2[aby]?\$", q) or re.match(r"^\$argon2", q) or re.match(r"^\$[156]\$", q):      return "hash"
@@ -6274,8 +6290,12 @@ class FileSystemProvider(AsyncSource):
        records = []
        for m in re.finditer(pattern, text):
            groups = m.groups()
+            val = groups[0] if len(groups) > 0 else ""
+            # Route to username if the value is not an email address
+            is_email = "@" in val
            records.append(self._rec(
-                email    = groups[0] if len(groups) > 0 else "",
+                email    = val if is_email else "",
+                username = val if not is_email else "",
                password = groups[1] if len(groups) > 1 else "",
                breach_name = self.name,
                data_types  = [self.name, "Credentials"],
@@ -6406,9 +6426,14 @@ class NoxSourceProvider(FileSystemProvider):
        payload = _sub(d.get("payload") or {})

        if method == "POST":
-            status, text, _ = await self._post(session, url,
-                                                json_data=payload or None,
-                                                headers=hdrs)
+            if isinstance(payload, str):
+                status, text, _ = await self._post(session, url,
+                                                    raw_body=payload,
+                                                    headers=hdrs)
+            else:
+                status, text, _ = await self._post(session, url,
+                                                    json_data=payload or None,
+                                                    headers=hdrs)
        else:
            status, text, _ = await self._get(session, url, headers=hdrs)

@@ -6575,8 +6600,8 @@ class SourceOrchestrator:
                    "output_type":           raw.get("output_type", []),
                    "pivot_types":           raw.get("pivot_types", []),
                    "confidence":            raw.get("confidence", 0.5),
-                    # payload_template → payload for POST sources
-                    "payload":               raw.get("payload_template") or raw.get("payload") or {},
+                    # payload_template → payload for POST sources; raw_payload takes precedence
+                    "payload":               raw.get("raw_payload") or raw.get("payload_template") or raw.get("payload") or {},
                    # Pass resolved slot keys so FileSystemProvider can use them
                    "_slot_keys":            slot_keys,
                    # Two-phase poll support
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "nox-cli"
-version = "1.0.4"
+version = "1.0.5"
 description = "Advanced Asynchronous Cyber Threat Intelligence Framework"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { text = "Apache-2.0" }
@@ -9,7 +9,7 @@ requirements = [

 setup(
    name="nox-cli",
-    version="1.0.4",
+    version="1.0.5",
    author="nox-project",
    description="Advanced Asynchronous Cyber Threat Intelligence Framework",
    long_description=Path("README.md").read_text(),
@@ -1,14 +1,16 @@
 {
    "name": "pypi_user",
    "category": "social",
-    "endpoint": "https://pypi.org/pypi/{target}/json",
-    "method": "GET",
+    "endpoint": "https://pypi.org/pypi",
+    "method": "POST",
    "requires_auth": false,
-    "selectors": {
-        "info": "$.info"
-    },
+    "selectors": {},
    "rate_limit": 1.0,
-    "headers": {},
+    "headers": {
+        "Content-Type": "text/xml"
+    },
+    "raw_payload": "<?xml version='1.0'?><methodCall><methodName>user_packages</methodName><params><param><value>{target}</value></param></params></methodCall>",
+    "regex_pattern": "<string>Owner</string></value>\\s*<value><string>([^<]+)</string>",
    "api_key_slots": [],
    "input_type": "username",
    "output_type": [
@@ -20,7 +22,7 @@
    ],
    "health_check_url": "https://pypi.org",
    "expected_status": 200,
-    "reliability_score": 5,
+    "reliability_score": 4,
    "backup_endpoints": [],
-    "confidence": 1.0
+    "confidence": 0.85
 }