Merge pull request #989 from MISP/codex/expansion-proofpoint-et-intelligence

Add Proofpoint ET Intelligence expansion module

Author: adulau

documentation/expansion/proofpoint_et_intelligence.md

@@ -0,0 +1,37 @@
+# Proofpoint ET Intelligence
+
+Enrich domains, IPv4 addresses and MD5 hashes using Proofpoint ET Intelligence.
+
+Uses the v1 API with Authorization header authentication. IP and domain queries return reputation plus passive DNS pivots; MD5 queries return sample details. IPv6 is rejected before querying because the service contract is IPv4. max_results defaults to 25 (1-100). Reputation scores are retained as context, including score zero, and do not change source IDS flags.
+
+Input: domain, hostname, ip-src, ip-dst, md5.
+
+Configuration: apikey, max_results. Required: apikey.
+
+Requests verify TLS, reject redirects and use a 30 second timeout. Credentials stay in authentication headers or request bodies. Service errors are reported without echoing credentials or response bodies.
+
+New attributes use `to_ids=false`; the source attribute retains its UUID and metadata. Results reference the source. Empty lookups return the source without fabricated enrichment.
+
+References:
+
+- https://eti.emergingthreats.net/
+
+Example request:
+
+```json
+{
+  "module": "proofpoint_et_intelligence",
+  "attribute": {
+    "type": "domain",
+    "value": "example.com",
+    "uuid": "32876e4a-916d-4935-9e51-7b69a76718f8"
+  },
+  "config": {
+    "apikey": "YOUR_API_KEY"
+  }
+}
+```
+
+Validation: `pytest tests/test_proofpoint_et_intelligence.py`. Tests use mocked provider responses; authenticated live API calls were not exercised.
+
+Sample `file_size` is mapped to the file object `size-in-bytes` relation, including zero-byte files.

misp_modules/modules/expansion/proofpoint_et_intelligence.py

@@ -0,0 +1,221 @@
+import json
+import re
+import xml.etree.ElementTree as ET
+from ipaddress import ip_address
+from urllib.parse import quote, urlsplit
+
+import requests
+from pymisp import MISPEvent, MISPObject
+
+misperrors = {"error": "Error"}
+mispattributes = {"input": ["domain", "hostname", "ip-src", "ip-dst", "md5"], "format": "misp_standard"}
+moduleconfig = ["apikey", "max_results"]
+moduleinfo = {
+    "version": "1",
+    "author": "MISP Project",
+    "name": "Proofpoint ET Intelligence",
+    "description": "Enrich domains, IPv4 addresses and MD5 hashes using Proofpoint ET Intelligence.",
+    "module-type": ["expansion"],
+    "logo": "",
+    "requirements": ["requests"],
+    "features": (
+        "Uses the v1 API with Authorization header authentication. IP and domain queries return reputation plus passive"
+        " DNS pivots; MD5 queries return sample details. IPv6 is rejected before querying because the service contract"
+        " is IPv4. max_results defaults to 25 (1-100). Reputation scores are retained as context, including score zero,"
+        " and do not change source IDS flags."
+    ),
+    "references": ["https://eti.emergingthreats.net/"],
+    "input": "domain, hostname, ip-src, ip-dst, md5 MISP attribute.",
+    "output": "Referenced enrichment objects and pivotable attributes.",
+}
+TIMEOUT = 30
+HASH_LENGTHS = {"md5": 32, "sha1": 40, "sha256": 64}
+
+
+class ModuleError(ValueError):
+    """An actionable error that contains no request credentials."""
+
+
+def _request(method, url, response_format="json", **kwargs):
+    try:
+        response = requests.request(method, url, timeout=TIMEOUT, allow_redirects=False, verify=True, **kwargs)
+    except requests.RequestException:
+        raise ModuleError("The service request failed or timed out.") from None
+    if response.status_code in (404, 204):
+        return None
+    if response.status_code not in (200, 201, 202):
+        raise ModuleError(f"The service returned HTTP {response.status_code}.")
+    if response_format == "bytes":
+        return response.content
+    if response_format == "xml":
+        if b"<!ENTITY" in response.content.upper() or re.search(b"<!DOCTYPE[^>]*\\[", response.content, re.I):
+            raise ModuleError("The service returned XML with a forbidden declaration.")
+        try:
+            return ET.fromstring(response.content)
+        except ET.ParseError:
+            raise ModuleError("The service returned invalid XML.") from None
+    try:
+        return response.json()
+    except ValueError:
+        raise ModuleError("The service returned invalid JSON.") from None
+
+
+def _domain(value):
+    if not isinstance(value, str):
+        raise ModuleError("Invalid domain or hostname.")
+    value = value.rstrip(".").encode("idna").decode("ascii").lower()
+    if len(value) > 253 or not all(
+        (re.fullmatch("[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", part) for part in value.split("."))
+    ):
+        raise ModuleError("Invalid domain or hostname.")
+    return value
+
+
+def _hash(kind, value):
+    if not isinstance(value, str) or not re.fullmatch("[0-9a-fA-F]{%d}" % HASH_LENGTHS[kind], value):
+        raise ModuleError(f"Invalid {kind} hash.")
+    return value.lower()
+
+
+def _value(attribute):
+    kind, value = (attribute["type"], attribute["value"].strip())
+    if kind in HASH_LENGTHS:
+        return _hash(kind, value)
+    if kind in ("ip-src", "ip-dst"):
+        return str(ip_address(value))
+    if kind in ("domain", "hostname"):
+        return _domain(value)
+    if kind == "url":
+        parts = urlsplit(value)
+        if parts.scheme not in ("http", "https") or not parts.hostname or parts.username or parts.password:
+            raise ModuleError("Invalid HTTP(S) URL.")
+    return value
+
+
+def _limit(config, key="max_results", default=25, maximum=100):
+    value = config.get(key)
+    if value is None or value == "":
+        return default
+    if isinstance(value, bool) or not str(value).isdigit() or (not 1 <= int(value) <= maximum):
+        raise ModuleError(f"{key} must be an integer between 1 and {maximum}.")
+    return int(value)
+
+
+def _annotation(event, attribute, title, data):
+    obj = MISPObject("annotation")
+    obj.comment = title
+    obj.add_attribute("text", value=json.dumps(data, sort_keys=True, indent=2), to_ids=False, disable_correlation=True)
+    obj.add_reference(attribute["uuid"], "describes")
+    event.add_object(obj)
+
+
+def _file(event, attribute, data, mapping=None):
+    mapping = mapping or {"md5": "md5", "sha1": "sha1", "sha256": "sha256"}
+    obj = MISPObject("file")
+    for kind, key in mapping.items():
+        if data.get(key):
+            obj.add_attribute(kind, value=_hash(kind, data[key]), to_ids=False)
+    for relation, key in (("filename", "filename"), ("mimetype", "mimetype"), ("size-in-bytes", "size")):
+        if data.get(key) is not None and data[key] != "":
+            obj.add_attribute(relation, value=data[key], to_ids=False)
+    if obj.attributes:
+        obj.add_reference(attribute["uuid"], "related-to")
+        event.add_object(obj)
+
+
+def _cap(event, attribute, rows, limit, more=False):
+    if len(rows) > limit or more:
+        _annotation(
+            event, attribute, moduleinfo["name"] + " result limit", {"limit": limit, "more_results_available": True}
+        )
+    return rows[:limit]
+
+
+def _results(event):
+    data = json.loads(event.to_json())
+    return {"results": {key: data.get(key, []) for key in ("Attribute", "Object")}}
+
+
+def handler(q=False):
+    if q is False:
+        return False
+    try:
+        request = json.loads(q)
+        if not isinstance(request, dict):
+            raise ModuleError("The request must be a JSON object.")
+        attribute = request.get("attribute")
+        if not isinstance(attribute, dict) or not all((attribute.get(k) for k in ("type", "value", "uuid"))):
+            raise ModuleError("An attribute containing type, value and uuid is required.")
+        if attribute["type"] not in mispattributes["input"]:
+            raise ModuleError("Unsupported attribute type.")
+        if not isinstance(attribute["value"], str) or not attribute["value"].strip():
+            raise ModuleError("The attribute value must be a nonempty string.")
+        config = request.get("config")
+        if config is None:
+            config = {}
+        if not isinstance(config, dict):
+            raise ModuleError("The config must be a JSON object.")
+        for key in ["apikey"]:
+            if not isinstance(config.get(key), str) or not config[key].strip():
+                raise ModuleError(f"Missing {key} configuration.")
+        event = MISPEvent()
+        event.add_attribute(**attribute)
+        return _expand(event, attribute, config)
+    except ModuleError as exc:
+        return {"error": str(exc)}
+    except (ValueError, TypeError, KeyError, OverflowError, OSError, AttributeError, IndexError):
+        return {"error": "Invalid request or service response."}
+
+
+def introspection():
+    return mispattributes
+
+
+def version():
+    return dict(moduleinfo, config=moduleconfig)
+
+
+def _payload(data):
+    if not isinstance(data, dict) or data.get("success") is not True or "response" not in data:
+        raise ModuleError("Proofpoint ET Intelligence rejected the lookup or returned an unexpected response.")
+    return data["response"]
+
+
+def _expand(event, attribute, config):
+    value, kind, limit = (_value(attribute), attribute["type"], _limit(config))
+    headers = {"Authorization": config["apikey"]}
+    if kind in ("ip-src", "ip-dst") and ip_address(value).version != 4:
+        raise ModuleError("Proofpoint ET Intelligence requires an IPv4 address.")
+    collection = "samples" if kind == "md5" else "ips" if kind in ("ip-src", "ip-dst") else "domains"
+    base = "https://api.emergingthreats.net/v1/" + collection + "/" + quote(value, safe="")
+    if kind == "md5":
+        data = _request("GET", base, headers=headers)
+        if data is None:
+            return _results(event)
+        detail = _payload(data)
+        if not isinstance(detail, dict) or _hash("md5", detail.get("md5sum", "")) != value:
+            raise ModuleError("Proofpoint ET Intelligence returned a different sample.")
+        _annotation(event, attribute, "Proofpoint sample details", detail)
+        _file(
+            event,
+            attribute,
+            dict(detail, size=detail.get("file_size", detail.get("size"))),
+            {"md5": "md5sum", "sha256": "sha256"},
+        )
+        return _results(event)
+    for suffix in ("reputation", "domains" if collection == "ips" else "ips"):
+        data = _request("GET", base + "/" + suffix, headers=headers)
+        if data is None:
+            continue
+        rows = _payload(data)
+        if not isinstance(rows, list) or not all((isinstance(row, dict) for row in rows)):
+            raise ModuleError("Proofpoint ET Intelligence returned invalid results.")
+        for row in _cap(event, attribute, rows, limit):
+            _annotation(event, attribute, "Proofpoint " + suffix, row)
+            if suffix == "domains" and row.get("domain"):
+                event.add_attribute("domain", _domain(row["domain"]), to_ids=False, comment="Proofpoint passive DNS")
+            if suffix == "ips" and row.get("ip"):
+                event.add_attribute(
+                    "ip-dst", str(ip_address(row["ip"])), to_ids=False, comment="Proofpoint passive DNS"
+                )
+    return _results(event)