diff --git a/README.md b/README.md index 58836728f..ccf298570 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [VirusTotal Collections Export](https://misp.github.io/misp-modules/export_mod/#virustotal-collections-export) - Creates a VT Collection from an event iocs. * [VirusTotal Graph Export](https://misp.github.io/misp-modules/export_mod/#virustotal-graph-export) - This module is used to create a VirusTotal Graph from a MISP event. * [YARA Rule Export](https://misp.github.io/misp-modules/export_mod/#yara-rule-export) - This module is used to export MISP events to YARA. +* [Zeek Intel Export](https://misp.github.io/misp-modules/export_mod/#zeek-intel-export) - Export validated MISP event and object indicators as a Zeek Intel Framework feed. ### Import Modules * [ANYRUN Sandbox Import](https://misp.github.io/misp-modules/import_mod/#anyrun-sandbox-import) - A module designed to retrieve an analysis report from the ANY.RUN Sandbox by its unique ID and extract results (such as verdict, malware tags, and IOCs), converting them into MISP attributes within your event. diff --git a/documentation/README.md b/documentation/README.md index bfaad23cb..c57f6963c 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -3953,6 +3953,31 @@ This module is used to export MISP events to YARA. ----- +#### [Zeek Intel Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/zeek_intel_export.py) + +Export validated MISP event and object indicators as a Zeek Intel Framework feed. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/zeek_intel_export.py)] + +- **features**: +>Produces a tab-separated Intel input file with native indicator types, escaped fields, stable source labels and MISP provenance. Supports addresses/subnets, domains, HTTP(S) URLs, emails, file names/hashes, certificate SHA-1 hashes and SSH MD5 fingerprints. Defaults to IDS-marked attributes and excludes deleted data. No Zeek connection or additional dependency is required. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/export_mod/zeek_intel_export.md). + +- **config**: +> - source +> - misp_url +> - include_non_ids + +- **input**: +>MISP events and object attributes in standard, bare or legacy export format. + +- **output**: +>Base64-encoded ASCII Zeek Intel TSV, with skipped counts and provenance in the API response. + +- **references**: +> - https://docs.zeek.org/en/current/frameworks/intel.html +> - https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html + +----- + ## Import Modules #### [ANYRUN Sandbox Import](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/import_mod/anyrun_sandbox_import.py) diff --git a/documentation/export_mod/zeek_intel_export.md b/documentation/export_mod/zeek_intel_export.md new file mode 100644 index 000000000..fc02cc7de --- /dev/null +++ b/documentation/export_mod/zeek_intel_export.md @@ -0,0 +1,200 @@ +# Zeek Intel Export + +`zeek_intel_export` converts supplied MISP event and object attributes into a Zeek +Intel Framework input file. It produces the native tab-separated ASCII format, +with a `#fields` header and these columns: + +```text +indicator indicator_type meta.source meta.desc meta.url +``` + +The separators in the actual file are **single tab characters**, not spaces. +No Zeek connection, credentials or extra Python dependencies are required. Zeek's +standard ingestion path is a file on its manager, so this implementation is an +export module; deployment remains with your existing file distribution process. +It does not install policy, reload sensors or publish through a network service. + +MISP also offers a built-in Bro/Zeek export. This module provides a standalone +misp-modules export with explicit validation, object traversal, deterministic +deduplication, escaped fields and provenance aggregation. It uses only Zeek's +built-in Intel fields and types; no JA3 or other extension package is assumed. + +## Configuration + +Enable **Zeek Intel Export** in MISP's export-module settings. Configure these +settings there, or supply them in `config` when calling `/query` directly: + +| Setting | Default | Meaning | +| --- | --- | --- | +| `source` | `misp-modules` | Stable Zeek metadata source label, a non-empty string of at most 128 characters. Use distinct labels for independent feeds. | +| `misp_url` | Empty | Optional HTTPS MISP base URL, including an installation subpath if needed. No credentials, query or fragment. Adds an event link when the indicator has one unambiguous event UUID or numeric ID. | +| `include_non_ids` | `false` | Explicitly include non-IDS attributes. Deleted events, objects and attributes remain excluded. | + +Booleans accept JSON booleans, integers 0/1 and strings `true`, `false`, `1`, `0`. +Empty settings use the documented defaults. Only supplied attributes are examined; +the module does not fetch additional data from MISP. Input can be a standard +`{"Event": {...}}` event, a bare event, a legacy envelope with collections alongside +`Event`, or a list of these events. + +## Supported types + +| MISP type | Zeek type | Conversion | +| --- | --- | --- | +| `ip-src`, `ip-dst` | `Intel::ADDR` or `Intel::SUBNET` | Validate IPv4/IPv6 and canonicalize addresses. CIDRs become canonical networks, including clearing host bits. IPv4-mapped IPv6 addresses/networks normalize to IPv4, matching Zeek's identity. Scoped IPv6 is excluded. | +| `domain`, `hostname` | `Intel::DOMAIN` | IDNA encode, lowercase, remove a trailing DNS dot and validate DNS labels and the 253-character normalized name limit. | +| `domain\|ip` | Domain plus address/subnet | Validate both components and emit two independent indicators. The original association is retained as provenance, rather than enforced as a conjunction. | +| `url` | `Intel::URL` | Require absolute HTTP(S), remove the fragment, decode request-target escapes once as bytes, and generate Zeek-compatible Host/port keys. Supply `/` for an empty path. Credentials and literal whitespace are excluded. | +| `email`, `email-src`, `email-dst`, `target-email` | `Intel::EMAIL` | Accept a bare address with an ASCII local part and valid IDNA domain. Display-name and quoted-local-part formats are excluded. | +| `filename`, `email-attachment`, `attachment` | `Intel::FILE_NAME` | Emit the filename, with characters escaped for the ASCII reader. Binary attachment contents are not included. | +| `md5`, `sha1`, `sha256`, `sha512` | `Intel::FILE_HASH` | Validate hexadecimal length and lowercase. | +| `filename\|md5`, `filename\|sha1`, `filename\|sha256`, `filename\|sha512`, `malware-sample` | `Intel::FILE_HASH` | Emit only the hash component. Split at the last pipe so filenames may contain pipes. No sample data is exported. | +| `x509-fingerprint-sha1` | `Intel::CERT_HASH` | Validate a SHA-1 fingerprint; accept plain hex or colon-separated octets and emit lowercase plain hex. | +| `ssh-fingerprint` | `Intel::PUBKEY_HASH` | Accept an MD5 fingerprint in plain hex or colon-separated octets, optionally prefixed `MD5:`, and emit lowercase colon-separated octets. SHA-256 SSH fingerprints are excluded. | + +Other types, including port-qualified composites, fuzzy/import hashes, user-agent +strings, JA3 and SHA-256 certificate fingerprints, are skipped. Port constraints +are not silently discarded. Unsupported and malformed attributes have separate +skipped counts. Invalid supported values do not prevent other eligible values +from being exported; malformed containers or configuration return an error. + +Zeek's Intel string store matches case-insensitively. This module folds ASCII +letters in indicator keys to lowercase and merges matching keys so their common +source label cannot overwrite another attribute's metadata. This includes URL +paths, filenames and email local parts: Zeek Intel does not provide case-sensitive +matching for them. Non-ASCII bytes are preserved, and original MISP values remain +in the provenance. This is not a case-sensitive URL or filename matching engine. + +URL keys follow Zeek's HTTP analyzer and `HTTP::build_url` observer. Percent escapes +in the path and query are decoded once as bytes before matching; `+` remains a +literal plus. For example, `http://example.org/a%2Fb?x=%41` becomes +`example.org/a/b?x=a`. Encoded `%25` is not recursively decoded. A literal fragment +is removed before decoding, so an encoded `%23` remains part of the request target. +An empty query delimiter (`/a?`) is retained. Zeek's legacy `%uHHHH` and doubled +percent handling are also supported. If the decoded target contains `://`, Zeek's +helper returns that target without a Host prefix; the exporter does the same. + +Zeek appends the responder port when it differs from 80, including when the Host +header already contains a port. A URL such as `http://example.org:8080/a` therefore +emits both `example.org:8080/a` and `example.org:8080:8080/a`, covering Host headers +without and with the port. Explicit port 80 similarly emits the Host forms with +and without `:80`. The responder port is inferred from the URL: 80 for HTTP and +443 for HTTPS when omitted. Consequently, one URL attribute can produce multiple +rows, each retaining the same provenance. Distinct decoded keys are deduplicated. + +TLS normally hides HTTPS paths; loading an HTTPS-derived URL does not make that +path observable on encrypted traffic. Reverse proxies, rewritten Host headers or +custom observers may report different strings; compare those observations with +your feed when configuring those deployments. + +## Escaping and provenance + +The file is ASCII, with non-ASCII bytes, controls, backslashes and `#` escaped +as `\xHH`. Zeek's ASCII reader restores the bytes. Literal `-` and `(empty)` strings +are escaped so they do not conflict with reader markers. Tabs/newlines in filenames +and metadata cannot create new columns, records or header directives. Optional +`meta.url` values use Zeek's unset marker `-`. Percent-encoded URL bytes that are +not valid UTF-8 are preserved in the feed without replacement. The API's display +of such keys uses backslash escapes; the base64 file contains the exact native key. + +Each unique `(Intel type, canonical indicator)` appears once. `meta.source` is the +configured namespace. `meta.desc` is a JSON list of all contributing MISP event, +attribute and object references, MISP types and original values. References and +rows are sorted for deterministic output. `meta.url` points to the event only when +all references identify the same valid event; otherwise it is unset. + +Event titles, comments, tags and binary data are not copied. Provenance can still +identify MISP events and contain original indicator values, so apply the event's +sharing and handling policy when selecting input and distributing the feed. +Existing `file`, `domain-ip` and `x509` object templates suffice; no new objects are +required. Object attributes use the same supported-type rules as event attributes. + +## API example + +POST to the misp-modules `/query` endpoint: + +```json +{ + "module": "zeek_intel_export", + "config": { + "source": "MISP production", + "misp_url": "https://misp.example" + }, + "data": [{ + "Event": { + "uuid": "f1ef0409-c80a-4012-b401-b7d1050f4ed4", + "Attribute": [{ + "uuid": "aa09819b-f2d8-4661-83ea-d4897cafebc8", + "type": "domain", + "value": "EXAMPLE.org.", + "to_ids": true + }] + } + }] +} +``` + +The response contains `response: []` and base64-encoded file contents in `data`. +Decode that field and save it as a `.intel` file. An empty selection produces a +valid header-only feed. The `summary` field contains row count and skipped counts; +`sources` contains the canonical indicators and their contributing references. +Warnings explain skipped attributes. MISP's normal download returns the file and +may not display these additional response fields. + +## Zeek setup and deployment + +Add this to the manager's `local.zeek`, using your deployed file path: + +```zeek +@load base/frameworks/intel +@load frameworks/intel/seen + +redef Intel::read_files += { "/opt/zeek/share/zeek/site/misp.intel" }; +``` + +`frameworks/intel/seen` loads the standard observation scripts, including the file +Intel types, so observed data can be checked against the feed. Configure and +deploy the policy through your normal Zeek administration. For a cluster, the +feed file is needed on the manager; Zeek distributes Intel state to workers. + +Install updates **atomically**: write a complete new file beside the destination, +set the appropriate permissions, and rename it over the old file on the same +filesystem. Do not stream partial output into the live feed. Zeek's input reader +detects file changes and rereads the feed. Use a distinct `source` namespace for +each independently maintained feed: reusing a source for the same indicator in +separate files can replace its metadata. + +Matching observations produce `intel.log` entries. A loaded indicator alone does +not guarantee a traffic match: analyzers and observation scripts must report it. +Default address observations cover established TCP connections; additional policy +is needed for other connection classes. File hashes require the corresponding +hash analyzers; SHA-512 in particular needs a custom observer/analyzer. Certificate +and SSH fingerprints likewise depend on the applicable protocol/file analysis. +The feed does not enable analyzers, create notices or enforce blocking. It omits +extension-only fields such as `meta.do_notice` and per-item expiry metadata. + +### Withdrawal and expiry + +Deleting a MISP attribute or removing a row from an updated feed does **not** +automatically withdraw the previously inserted indicator. Zeek's standard Intel +input handler inserts reread items; it does not reconcile absent rows. The default +`Intel::item_expiration` is disabled, and changing a feed's `source` can leave old +metadata behind. Plan withdrawal through your Zeek policy, using explicit Intel +removal, a configured expiration interval with an appropriate `Intel::item_expired` +hook, or a controlled restart with the intended complete feeds. This module emits +no removal instructions and does not set an expiry time. + +## Validation + +Tests cover supported mappings, objects, case-insensitive deduplication, provenance, +escaping, malformed input and the module server's discovery/query endpoints. +Optional native tests load the exported file into Zeek, exercise the real Intel +matcher, and replay HTTP packet captures through the standard URL observer. Set +`ZEEK` to an installed Zeek executable (or place `zeek` on PATH) to run them; they +are skipped when no runtime is available. Native validation was performed with +Zeek 8.0.5, including UTF-8, control characters, reader markers and +literal backslash escape sequences. No running sensor was changed. + +References: [Intel Framework](https://docs.zeek.org/en/current/frameworks/intel.html), +[native Intel types](https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html), +[ASCII reader](https://docs.zeek.org/en/current/scripts/base/frameworks/input/readers/ascii.zeek.html), +and [Intel file loading and atomic updates](https://docs.zeek.org/en/current/scripts/base/frameworks/intel/input.zeek.html). diff --git a/documentation/mkdocs/export_mod.md b/documentation/mkdocs/export_mod.md index 2681ccb99..d5290011f 100644 --- a/documentation/mkdocs/export_mod.md +++ b/documentation/mkdocs/export_mod.md @@ -477,3 +477,28 @@ This module is used to export MISP events to YARA. >yara-python python library ----- + +#### [Zeek Intel Export](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/zeek_intel_export.py) + +Export validated MISP event and object indicators as a Zeek Intel Framework feed. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/export_mod/zeek_intel_export.py)] + +- **features**: +>Produces a tab-separated Intel input file with native indicator types, escaped fields, stable source labels and MISP provenance. Supports addresses/subnets, domains, HTTP(S) URLs, emails, file names/hashes, certificate SHA-1 hashes and SSH MD5 fingerprints. Defaults to IDS-marked attributes and excludes deleted data. No Zeek connection or additional dependency is required. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/export_mod/zeek_intel_export.md). + +- **config**: +> - source +> - misp_url +> - include_non_ids + +- **input**: +>MISP events and object attributes in standard, bare or legacy export format. + +- **output**: +>Base64-encoded ASCII Zeek Intel TSV, with skipped counts and provenance in the API response. + +- **references**: +> - https://docs.zeek.org/en/current/frameworks/intel.html +> - https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html + +----- diff --git a/documentation/mkdocs/index.md b/documentation/mkdocs/index.md index 6523d8fab..7079156da 100644 --- a/documentation/mkdocs/index.md +++ b/documentation/mkdocs/index.md @@ -164,6 +164,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [VirusTotal Collections Export](https://misp.github.io/misp-modules/export_mod/#virustotal-collections-export) - Creates a VT Collection from an event iocs. * [VirusTotal Graph Export](https://misp.github.io/misp-modules/export_mod/#virustotal-graph-export) - This module is used to create a VirusTotal Graph from a MISP event. * [YARA Rule Export](https://misp.github.io/misp-modules/export_mod/#yara-rule-export) - This module is used to export MISP events to YARA. +* [Zeek Intel Export](https://misp.github.io/misp-modules/export_mod/#zeek-intel-export) - Export validated MISP event and object indicators as a Zeek Intel Framework feed. ### Import Modules * [ANYRUN Sandbox Import](https://misp.github.io/misp-modules/import_mod/#anyrun-sandbox-import) - A module designed to retrieve an analysis report from the ANY.RUN Sandbox by its unique ID and extract results (such as verdict, malware tags, and IOCs), converting them into MISP attributes within your event. diff --git a/misp_modules/modules/export_mod/zeek_intel_export.py b/misp_modules/modules/export_mod/zeek_intel_export.py new file mode 100644 index 000000000..e1109dcb9 --- /dev/null +++ b/misp_modules/modules/export_mod/zeek_intel_export.py @@ -0,0 +1,350 @@ +"""Export MISP indicators as a Zeek Intel Framework ASCII input file.""" + +import base64 +import ipaddress +import json +import re +import uuid +from collections import Counter +from urllib.parse import urlsplit + +moduleinfo = { + "version": "1.0", + "author": "MISP Project", + "name": "Zeek Intel Export", + "description": "Export validated MISP event and object indicators as a Zeek Intel Framework feed.", + "module-type": ["export"], + "logo": "", + "requirements": [], + "features": ( + "Produces a tab-separated Intel input file with native indicator types, escaped fields, stable source labels" + " and MISP provenance. Supports addresses/subnets, domains, HTTP(S) URLs, emails, file names/hashes," + " certificate SHA-1 hashes and SSH MD5 fingerprints. Defaults to IDS-marked attributes and excludes deleted" + " data. No Zeek connection or additional dependency is required. See the [usage" + " guide](https://github.com/MISP/misp-modules/blob/main/documentation/export_mod/zeek_intel_export.md)." + ), + "references": [ + "https://docs.zeek.org/en/current/frameworks/intel.html", + "https://docs.zeek.org/en/current/scripts/base/frameworks/intel/main.zeek.html", + ], + "input": "MISP events and object attributes in standard, bare or legacy export format.", + "output": "Base64-encoded ASCII Zeek Intel TSV, with skipped counts and provenance in the API response.", +} +moduleconfig = ["source", "misp_url", "include_non_ids"] +FIELDS = ("indicator", "indicator_type", "meta.source", "meta.desc", "meta.url") +HASH_LENGTHS = {"md5": 32, "sha1": 40, "sha256": 64, "sha512": 128} +ASCII_LOWER = str.maketrans("ABCDEFGHIJKLMNOPQRSTUVWXYZ", "abcdefghijklmnopqrstuvwxyz") +EMAIL_TYPES = {"email", "email-src", "email-dst", "target-email"} +FILENAME_TYPES = {"filename", "email-attachment", "attachment"} +SUPPORTED_TYPES = ( + { + "ip-src", + "ip-dst", + "domain", + "hostname", + "domain|ip", + "url", + "x509-fingerprint-sha1", + "ssh-fingerprint", + "malware-sample", + } + | EMAIL_TYPES + | FILENAME_TYPES + | HASH_LENGTHS.keys() + | {f"filename|{kind}" for kind in HASH_LENGTHS} +) + + +def _boolean(value, name): + if value is None or value == "": + return False + if isinstance(value, (bool, int)) and value in (0, 1): + return bool(value) + if isinstance(value, str) and value.lower() in ("true", "false", "1", "0"): + return value.lower() in ("true", "1") + raise ValueError(f"{name} must be true, false, 1 or 0.") + + +def _records(container, key): + records = container.get(key, []) + if not isinstance(records, list) or any(not isinstance(record, dict) for record in records): + raise ValueError(f"{key} must be a list of objects.") + return records + + +def _settings(config): + if config is None: + config = {} + if not isinstance(config, dict): + raise ValueError("config must be an object.") + source = config.get("source") + if source is None or source == "": + source = "misp-modules" + if not isinstance(source, str) or not source.strip() or len(source) > 128: + raise ValueError("source must be a non-empty string of at most 128 characters.") + source.encode("utf-8") + base = config.get("misp_url") + if base is None or base == "": + base = None + else: + if not isinstance(base, str): + raise ValueError("misp_url must be an HTTPS base URL.") + parsed = urlsplit(base) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + or any(ord(char) <= 32 or ord(char) == 127 for char in base) + ): + raise ValueError("misp_url must be HTTPS without credentials, a query or a fragment.") + parsed.port + base = base.rstrip("/") + return source, base, _boolean(config.get("include_non_ids"), "include_non_ids") + + +def _domain(value): + domain = value.removesuffix(".").encode("idna").decode("ascii").lower() + if len(domain) > 253 or any( + not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) for label in domain.split(".") + ): + raise ValueError("Invalid DNS name.") + return domain + + +def _fingerprint(value, length): + if re.fullmatch(r"[0-9a-fA-F]{" + str(length) + r"}", value): + return value.lower() + if re.fullmatch(r"(?:[0-9a-fA-F]{2}:){" + str(length // 2 - 1) + r"}[0-9a-fA-F]{2}", value): + return value.replace(":", "").lower() + raise ValueError("Invalid fingerprint.") + + +def _unescape_uri(value): + """Mirror Zeek HTTP's byte-oriented, single-pass unescape_URI routine.""" + raw, decoded, index = value.encode("utf-8"), bytearray(), 0 + while index < len(raw): + remaining = raw[index : index + 6] + if remaining.startswith(b"%%") and len(remaining) >= 3: + index += 1 # Zeek ignores the first '%' in a doubled escape. + continue + if re.match(rb"%[0-9a-fA-F]{2}", remaining): + decoded.append(int(remaining[1:3], 16)) + index += 3 + elif re.match(rb"%u[0-9a-fA-F]{4}", remaining): + high, low = int(remaining[2:4], 16), int(remaining[4:6], 16) + if high: + decoded.append(high) + decoded.append(low) + index += 6 + else: + decoded.append(raw[index]) + index += 1 + return bytes(decoded) + + +def _urls(value): + parsed = urlsplit(value) + if ( + parsed.scheme not in ("http", "https") + or not parsed.hostname + or parsed.username is not None + or parsed.password is not None + or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value) + ): + raise ValueError("URL must be absolute HTTP(S), without credentials or whitespace.") + port = parsed.port + host = parsed.netloc.rsplit(":", 1)[0] if port is not None else parsed.netloc + responder_port = port if port is not None else (80 if parsed.scheme == "http" else 443) + target = parsed.path or "/" + if "?" in value.split("#", 1)[0]: + target += "?" + parsed.query + target = _unescape_uri(target) + # HTTP::build_url returns request targets containing '://' without a Host prefix. + if b"://" in target: + return [("Intel::URL", target)] + authorities = {host.encode("utf-8")} + if port is not None or responder_port != 80: + authorities.add(f"{host}:{responder_port}".encode("utf-8")) + if port is not None: + authorities.add(parsed.netloc.encode("utf-8")) + # The observer appends the responder port even when Host already includes it. + suffix = f":{responder_port}".encode("ascii") if responder_port != 80 else b"" + return [("Intel::URL", authority + suffix + target) for authority in sorted(authorities)] + + +def _atomic(kind, value): + value.encode("utf-8") + if kind in FILENAME_TYPES: + return "Intel::FILE_NAME", value + if any(ord(char) < 32 or ord(char) == 127 for char in value): + raise ValueError("Control characters are unsupported for this indicator type.") + if kind in HASH_LENGTHS: + if not re.fullmatch(rf"[0-9a-fA-F]{{{HASH_LENGTHS[kind]}}}", value): + raise ValueError("Invalid file hash.") + return "Intel::FILE_HASH", value.lower() + if kind in ("ip-src", "ip-dst"): + if "%" in value: + raise ValueError("Scoped IPv6 is unsupported.") + if "/" in value: + network = ipaddress.ip_network(value, strict=False) + if network.version == 6 and network.network_address.ipv4_mapped is not None: + network = ipaddress.IPv4Network((network.network_address.ipv4_mapped, network.prefixlen - 96)) + return "Intel::SUBNET", str(network) + address = ipaddress.ip_address(value) + if address.version == 6 and address.ipv4_mapped is not None: + address = address.ipv4_mapped + return "Intel::ADDR", str(address) + if kind in ("domain", "hostname"): + return "Intel::DOMAIN", _domain(value) + if kind in EMAIL_TYPES: + local, separator, domain = value.rpartition("@") + if not separator or not re.fullmatch(r"[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+", local): + raise ValueError("Unsupported email address.") + return "Intel::EMAIL", local + "@" + _domain(domain) + if kind == "x509-fingerprint-sha1": + return "Intel::CERT_HASH", _fingerprint(value, 40) + if kind == "ssh-fingerprint": + digest = _fingerprint(re.sub(r"^MD5:", "", value, flags=re.IGNORECASE), 32) + return "Intel::PUBKEY_HASH", ":".join(digest[index : index + 2] for index in range(0, 32, 2)) + raise ValueError("Unsupported atomic indicator.") + + +def _indicators(kind, value): + if kind == "url": + return _urls(value) + if "|" not in kind and kind != "malware-sample": + return [_atomic(kind, value)] + first, separator, second = value.rpartition("|") + if not separator or not first.strip() or not second.strip(): + raise ValueError("Invalid composite.") + if kind == "domain|ip": + return [_atomic("domain", first), _atomic("ip-dst", second)] + return [_atomic("md5" if kind == "malware-sample" else kind.split("|")[1], second)] + + +def _collect(data, include_non_ids): + if isinstance(data, dict): + data = [data] + if not isinstance(data, list) or any(not isinstance(envelope, dict) for envelope in data): + raise ValueError("data must contain an event or list of events.") + indicators, skipped = {}, Counter() + for envelope in data: + event = envelope.get("Event", envelope) + if not isinstance(event, dict): + raise ValueError("Event must be an object.") + if _boolean(event.get("deleted"), "deleted"): + continue + collections = dict(event, **envelope) + attributes = [(attribute, {}) for attribute in _records(collections, "Attribute")] + for obj in _records(collections, "Object"): + if not _boolean(obj.get("deleted"), "deleted"): + attributes.extend((attribute, obj) for attribute in _records(obj, "Attribute")) + for attribute, obj in attributes: + if _boolean(attribute.get("deleted"), "deleted") or ( + not _boolean(attribute.get("to_ids"), "to_ids") and not include_non_ids + ): + skipped["deleted or non-IDS"] += 1 + continue + kind, value = attribute.get("type"), attribute.get("value") + if not isinstance(kind, str) or kind not in SUPPORTED_TYPES: + skipped["unsupported"] += 1 + continue + try: + if not isinstance(value, str) or not value.strip(): + raise ValueError("Empty value.") + atoms = _indicators(kind, value) + except ValueError: + skipped["malformed"] += 1 + continue + ref = { + "event": str(event.get("uuid") or event.get("id") or "unknown"), + "attribute": str(attribute.get("uuid") or attribute.get("id") or "unknown"), + "misp_type": kind, + "original_value": value, + } + if obj: + ref["object"] = str(obj.get("uuid") or obj.get("id") or obj.get("name") or "unknown") + ref["object_relation"] = str(attribute.get("object_relation") or "") + # Zeek's Intel string store is case-insensitive. Merge those keys so + # identical source labels cannot overwrite another attribute's provenance. + for atom_type, indicator in atoms: + atom = atom_type, ( + indicator.lower() if isinstance(indicator, bytes) else indicator.translate(ASCII_LOWER) + ) + refs = indicators.setdefault(atom, []) + if ref not in refs: + refs.append(ref) + return indicators, dict(skipped) + + +def _escape(value): + if value is None: + return "-" + encoded = value if isinstance(value, bytes) else value.encode("utf-8") + escaped = "".join(chr(byte) if 32 <= byte <= 126 and byte not in (35, 92) else f"\\x{byte:02x}" for byte in encoded) + if escaped in ("-", "(empty)"): + escaped = f"\\x{encoded[0]:02x}" + escaped[1:] + return escaped + + +def _event_url(base, refs): + if base is None or len(events := {ref["event"] for ref in refs}) != 1: + return None + identifier = next(iter(events)) + if re.fullmatch(r"[1-9][0-9]*", identifier): + return f"{base}/events/view/{identifier}" + try: + identifier = str(uuid.UUID(identifier)) + except ValueError: + return None + return f"{base}/events/view/{identifier}" + + +def handler(q=False): + if not q: + return False + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ValueError("Request must be an object.") + source, base, include_non_ids = _settings(request.get("config")) + indicators, skipped = _collect(request.get("data"), include_non_ids) + lines, provenance = ["#fields\t" + "\t".join(FIELDS)], [] + for (kind, indicator), refs in sorted(indicators.items()): + refs = sorted(refs, key=lambda ref: json.dumps(ref, sort_keys=True)) + description = json.dumps(refs, sort_keys=True, ensure_ascii=False) + row = (indicator, kind, source, description, _event_url(base, refs)) + lines.append("\t".join(_escape(field) for field in row)) + displayed = ( + indicator.decode("utf-8", errors="backslashreplace") if isinstance(indicator, bytes) else indicator + ) + provenance.append({"indicator": displayed, "indicator_type": kind, "sources": refs}) + output = ("\n".join(lines) + "\n").encode("ascii") + except (ValueError, TypeError) as exc: + return {"error": f"Zeek intel export: {exc}"} + result = { + "response": [], + "data": base64.b64encode(output).decode("ascii"), + "summary": {"count": len(indicators), "skipped": skipped}, + "sources": provenance, + } + if skipped: + result["warnings"] = [f"Skipped {count} {reason} attribute(s)." for reason, count in sorted(skipped.items())] + return result + + +def introspection(): + return { + "input": sorted(SUPPORTED_TYPES), + "inputSource": ["event", "event-collection"], + "outputFileExtension": "intel", + "responseType": "text/plain", + "userConfig": {}, + } + + +def version(): + return dict(moduleinfo, config=moduleconfig) diff --git a/tests/test_zeek_intel_export.py b/tests/test_zeek_intel_export.py new file mode 100644 index 000000000..45f0fb55c --- /dev/null +++ b/tests/test_zeek_intel_export.py @@ -0,0 +1,624 @@ +import base64 +import copy +import json +import os +import re +import shutil +import socket +import struct +import subprocess + +import pytest +from pymisp import MISPObject +from tornado.testing import AsyncHTTPTestCase +from tornado.web import Application + +from misp_modules import __main__ as service +from misp_modules.modules.export_mod import zeek_intel_export as export + +EVENT_UUID = "f1ef0409-c80a-4012-b401-b7d1050f4ed4" + + +def attribute(kind="domain", value="example.org", **metadata): + return {"type": kind, "value": value, "to_ids": True, "uuid": "attribute-1", **metadata} + + +def event(attributes=None, objects=None, **metadata): + return {"uuid": EVENT_UUID, "Attribute": attributes or [], "Object": objects or [], **metadata} + + +def run(data=None, **config): + result = export.handler(json.dumps({"data": event([attribute()]) if data is None else data, "config": config})) + assert "error" not in result, result + return result + + +def decode_field(field): + if field == b"-": + return None + return re.sub(rb"\\x([0-9a-fA-F]{2})", lambda match: bytes([int(match[1], 16)]), field).decode("utf-8") + + +def rows(result): + lines = base64.b64decode(result["data"]).splitlines() + assert lines[0] == b"#fields\tindicator\tindicator_type\tmeta.source\tmeta.desc\tmeta.url" + assert all(len(line.split(b"\t")) == 5 for line in lines[1:]) + return [dict(zip(export.FIELDS, map(decode_field, line.split(b"\t")))) for line in lines[1:]] + + +@pytest.mark.parametrize( + "kind,value,intel_type,indicator", + [ + ("ip-src", "192.0.2.1", "ADDR", "192.0.2.1"), + ("ip-dst", "2001:0DB8:0000::1", "ADDR", "2001:db8::1"), + ("ip-dst", "::ffff:c000:201", "ADDR", "192.0.2.1"), + ("ip-src", "::ffff:c000:200/120", "SUBNET", "192.0.2.0/24"), + ("ip-src", "192.0.2.7/24", "SUBNET", "192.0.2.0/24"), + ("ip-dst", "2001:db8:1::1/48", "SUBNET", "2001:db8:1::/48"), + ("domain", "EXAMPLE.org.", "DOMAIN", "example.org"), + ("hostname", "bücher.example", "DOMAIN", "xn--bcher-kva.example"), + ("url", "HTTP://Example.org/Case?q=One#fragment", "URL", "example.org/case?q=one"), + ("url", "https://example.org:8443/A?b=C", "URL", {"example.org:8443/a?b=c", "example.org:8443:8443/a?b=c"}), + ("url", "https://example.org", "URL", {"example.org:443/", "example.org:443:443/"}), + ("url", "http://example.org?x=1", "URL", "example.org/?x=1"), + ("url", "http://[2001:db8::1]:8080/Path", "URL", {"[2001:db8::1]:8080/path", "[2001:db8::1]:8080:8080/path"}), + *[(kind, "A" * length, "FILE_HASH", "a" * length) for kind, length in export.HASH_LENGTHS.items()], + *[ + (f"filename|{kind}", "a|b.exe|" + "A" * length, "FILE_HASH", "a" * length) + for kind, length in export.HASH_LENGTHS.items() + ], + ("malware-sample", "file.exe|" + "A" * 32, "FILE_HASH", "a" * 32), + *[(kind, "User@EXAMPLE.org", "EMAIL", "user@example.org") for kind in export.EMAIL_TYPES], + ("email-src", "person@bücher.example", "EMAIL", "person@xn--bcher-kva.example"), + *[(kind, "MyFile.EXE", "FILE_NAME", "myfile.exe") for kind in export.FILENAME_TYPES], + ("x509-fingerprint-sha1", "AB" * 20, "CERT_HASH", "ab" * 20), + ("x509-fingerprint-sha1", ":".join(["AB"] * 20), "CERT_HASH", "ab" * 20), + ("ssh-fingerprint", "AB" * 16, "PUBKEY_HASH", ":".join(["ab"] * 16)), + ("ssh-fingerprint", "MD5:" + ":".join(["AB"] * 16), "PUBKEY_HASH", ":".join(["ab"] * 16)), + ], +) +def test_supported_types_and_normalization(kind, value, intel_type, indicator): + result = run(event([attribute(kind, value)])) + expected = {indicator} if isinstance(indicator, str) else indicator + output_rows = rows(result) + assert {row["indicator"] for row in output_rows} == expected + for row in output_rows: + assert row["indicator_type"] == "Intel::" + intel_type + assert row["meta.source"] == "misp-modules" and row["meta.url"] is None + assert json.loads(row["meta.desc"])[0]["original_value"] == value + assert result["summary"] == {"count": len(expected), "skipped": {}} + + +def test_domain_ip_emits_two_independent_indicators(): + result = run(event([attribute("domain|ip", "EXAMPLE.org|192.0.2.1")])) + assert {(row["indicator_type"], row["indicator"]) for row in rows(result)} == { + ("Intel::DOMAIN", "example.org"), + ("Intel::ADDR", "192.0.2.1"), + } + assert all(json.loads(row["meta.desc"])[0]["misp_type"] == "domain|ip" for row in rows(result)) + + +@pytest.mark.parametrize( + "url,expected", + [ + ("http://example.org/a%2Fb?x=%41+%2B", {"example.org/a/b?x=a++"}), + ("http://example.org/a%252Fb", {"example.org/a%2fb"}), + ("http://example.org/a%20b", {"example.org/a b"}), + ("http://example.org/a%23b?x=%3F", {"example.org/a#b?x=?"}), + ("http://example.org/a?", {"example.org/a?"}), + ("http://example.org:8080/a", {"example.org:8080/a", "example.org:8080:8080/a"}), + ("http://example.org:80/a", {"example.org/a", "example.org:80/a"}), + ("https://example.org/a", {"example.org:443/a", "example.org:443:443/a"}), + ("http://[2001:db8::1]:8080/a", {"[2001:db8::1]:8080/a", "[2001:db8::1]:8080:8080/a"}), + ], +) +def test_url_keys_match_decoded_targets_and_host_header_port_forms(url, expected): + result = run(event([attribute("url", url)])) + assert {row["indicator"] for row in rows(result)} == expected + assert all(json.loads(row["meta.desc"])[0]["original_value"] == url for row in rows(result)) + + +@pytest.mark.parametrize( + "target,expected", + [ + ("/%FF%00%09%5C", b"example.org/\xff\x00\t\\"), + ("/%%41", b"example.org/a"), + ("/%u0041%u00FF", b"example.org/a\xff"), + ("/%GG%a%", b"example.org/%gg%a%"), + ("/jump?next=http%3A%2F%2Fother.example", b"/jump?next=http://other.example"), + ], +) +def test_url_escape_decoding_preserves_native_bytes_without_injecting_rows(target, expected): + result = run(event([attribute("url", "http://example.org" + target)])) + raw = base64.b64decode(result["data"]) + assert len(raw.splitlines()) == 2 + fields = raw.splitlines()[1].split(b"\t") + assert len(fields) == 5 + decoded = re.sub(rb"\\x([0-9a-fA-F]{2})", lambda match: bytes([int(match[1], 16)]), fields[0]) + assert decoded == expected + # API provenance remains JSON-serializable even when the native key is not UTF-8. + json.dumps(result, ensure_ascii=False).encode("utf-8") + + +def _http_trace(requests): + """Build an offline Ethernet/IPv4/TCP trace without extra test dependencies.""" + trace = bytearray(struct.pack("> 16: + total = (total & 0xFFFF) + (total >> 16) + return (~total) & 0xFFFF + + index = 0 + + def packet(source_port, port, seq, ack, flags, payload=b"", reply=False): + nonlocal index + source, destination = socket.inet_aton("192.0.2.1"), socket.inet_aton("198.51.100.1") + if reply: + source, destination = destination, source + source_port, port = port, source_port + tcp = struct.pack("!HHIIBBHHH", source_port, port, seq, ack, 5 << 4, flags, 65535, 0, 0) + pseudo = source + destination + struct.pack("!BBH", 0, 6, len(tcp) + len(payload)) + tcp = tcp[:16] + struct.pack("!H", checksum(pseudo + tcp + payload)) + tcp[18:] + ip = struct.pack( + "!BBHHHBBH4s4s", 0x45, 0, 20 + len(tcp) + len(payload), index, 0, 64, 6, 0, source, destination + ) + ip = ip[:10] + struct.pack("!H", checksum(ip)) + ip[12:] + ethernet = bytes.fromhex("00112233445566778899aabb0800") + frame = ethernet + ip + tcp + payload + trace.extend(struct.pack(""), + ("email-src", "two@@example.org"), + ("md5", "bad"), + ("sha256", "g" * 64), + ("filename|sha1", "file.exe|bad"), + ("domain|ip", "example.org|not-ip"), + ("domain|ip", "bad..example|192.0.2.1"), + ("ssh-fingerprint", "SHA256:" + "A" * 43), + ("x509-fingerprint-sha1", "ab::" * 20), + ("filename", ""), + ("filename", "\ud800"), + ("domain", None), + ], +) +def test_malformed_supported_values_are_counted_and_skipped(kind, value): + result = run(event([attribute(kind, value)])) + assert result["summary"] == {"count": 0, "skipped": {"malformed": 1}} + assert rows(result) == [] and "warnings" in result + + +@pytest.mark.parametrize( + "kind", + [ + "ip-src|port", + "ip-dst|port", + "hostname|port", + "ja3-fingerprint-md5", + "user-agent", + "x509-fingerprint-sha256", + "imphash", + "yara", + ], +) +def test_unsupported_types_are_not_mapped_to_misleading_intel_types(kind): + assert run(event([attribute(kind, "unmapped")]))["summary"]["skipped"] == {"unsupported": 1} + + +@pytest.mark.parametrize( + "config", + [ + [], + {"source": []}, + {"source": False}, + {"source": " "}, + {"source": "a" * 129}, + {"source": "\ud800"}, + {"include_non_ids": "maybe"}, + {"include_non_ids": 2}, + {"misp_url": False}, + *[ + {"misp_url": url} + for url in [ + "http://misp.example", + "https://user:secret@misp.example", + "https://@misp.example", + "https://misp.example/?q=x", + "https://misp.example/#x", + "https://misp.example:bad", + "https://misp.example\n", + ] + ], + ], +) +def test_invalid_configuration_returns_error_without_file(config): + result = export.handler(json.dumps({"data": [], "config": config})) + assert "error" in result and "data" not in result + + +@pytest.mark.parametrize( + "data", [None, "bad", [1], {"Event": "bad"}, {"Attribute": {}}, {"Object": [1]}, event([attribute(to_ids="maybe")])] +) +def test_invalid_input_returns_error(data): + assert "error" in export.handler(json.dumps({"data": data})) + + +def test_empty_input_is_a_valid_header_only_feed_and_request_contract_is_stable(): + assert rows(run([])) == [] and run([])["summary"]["count"] == 0 + assert export.handler() is False + for payload in ["invalid JSON", "[]", "null"]: + assert "error" in export.handler(payload) + assert export.introspection()["outputFileExtension"] == "intel" + assert export.introspection()["responseType"] == "text/plain" + assert export.version()["module-type"] == ["export"] + + +def test_native_zeek_loads_and_matches_exported_indicators(tmp_path): + zeek = os.getenv("ZEEK") or shutil.which("zeek") + if not zeek: + pytest.skip("Set ZEEK to a Zeek executable to exercise the native reader and Intel matcher.") + samples = [ + ("ip-dst", "192.0.2.1"), + ("ip-src", "2001:db8::1"), + ("ip-dst", "198.51.100.5/24"), + ("domain", "EXAMPLE.org."), + ("url", "http://example.org/Case?q=One#fragment"), + ("email-src", "Person@example.org"), + ("md5", "a" * 32), + ("sha256", "b" * 64), + ("filename", r"C:\Temp\Bad\x09.exe"), + ("filename", "\t#fields\nmyfile.exe"), + ("filename", "-"), + ("filename", "(empty)"), + ("filename", "файл.txt"), + ("x509-fingerprint-sha1", "ab" * 20), + ("ssh-fingerprint", "ab" * 16), + ] + source = "MISP\\literal\ttab\nnewline" + result = run( + event([attribute(kind, value, uuid=f"attribute-{index}") for index, (kind, value) in enumerate(samples)]), + source=source, + ) + expected = rows(result) + feed = tmp_path / "feed.intel" + feed.write_bytes(base64.b64decode(result["data"])) + script = tmp_path / "check.zeek" + script.write_text(""" +@load base/frameworks/intel +@load frameworks/intel/seen +redef exit_only_after_terminate = T; +redef InputAscii::fail_on_invalid_lines = T; +redef InputAscii::fail_on_file_problem = T; +redef Intel::read_files = { "FEED_PATH" }; +global matched: count = 0; +global loaded_file: file; +event zeek_init() &priority=10 + { + loaded_file = open("loaded.jsonl"); + enable_raw_output(loaded_file); + } +event Intel::read_entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item) &priority=-5 + { + print loaded_file, fmt("%s\\n", to_json(item)); + if ( item$indicator_type == Intel::ADDR ) + Intel::seen([$host=to_addr(item$indicator), $where=Intel::IN_ANYWHERE]); + else if ( item$indicator_type == Intel::SUBNET ) + { + local parts = split_string(item$indicator, /\\//); + Intel::seen([$host=to_addr(parts[0]), $where=Intel::IN_ANYWHERE]); + } + else + Intel::seen([$indicator=item$indicator, $indicator_type=item$indicator_type, $where=Intel::IN_ANYWHERE]); + } +event Intel::match(s: Intel::Seen, items: set[Intel::Item]) + { + ++matched; + } +event Input::end_of_data(name: string, source: string) + { + terminate(); + } +event zeek_done() + { + close(loaded_file); + print fmt("MATCHED %d", matched); + } +""".replace("FEED_PATH", str(feed))) + process = subprocess.run([zeek, "-b", str(script)], cwd=tmp_path, text=True, capture_output=True, timeout=40) + assert process.returncode == 0, process.stdout + process.stderr + assert "error" not in process.stderr.lower() and "warning" not in process.stderr.lower(), process.stderr + loaded = [json.loads(line) for line in (tmp_path / "loaded.jsonl").read_text().splitlines()] + assert len(loaded) == len(expected) == len(samples), repr(process.stdout) + assert f"MATCHED {len(samples)}" in process.stdout + for actual, wanted in zip(sorted(loaded, key=lambda item: (item["indicator_type"], item["indicator"])), expected): + assert actual["indicator"] == wanted["indicator"] and actual["indicator_type"] == wanted["indicator_type"] + assert actual["meta"]["source"] == source + assert json.loads(actual["meta"]["desc"]) == json.loads(wanted["meta.desc"]) + + +class TestZeekExportHTTP(AsyncHTTPTestCase): + def get_app(self): + self.previous, self.cache = service.MODULES_HANDLERS.copy(), service.ListModules.CACHE + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update({"zeek_intel_export": export, "type:zeek_intel_export": "export_mod"}) + service.ListModules.CACHE = None + return Application([(r"/modules", service.ListModules), (r"/query", service.QueryModule)]) + + def tearDown(self): + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update(self.previous) + service.ListModules.CACHE = self.cache + super().tearDown() + + def test_discovery_and_query(self): + descriptor = json.loads(self.fetch("/modules").body)[0] + assert descriptor["name"] == "zeek_intel_export" and descriptor["type"] == "export_mod" + result = self.fetch( + "/query", + method="POST", + headers={"Content-Type": "application/json"}, + body=json.dumps({"module": "zeek_intel_export", "data": [event([attribute()])]}), + ) + assert result.code == 200 and rows(json.loads(result.body))[0]["indicator_type"] == "Intel::DOMAIN" + + def test_percent_encoded_binary_url_has_a_valid_json_api_response(self): + result = self.fetch( + "/query", + method="POST", + headers={"Content-Type": "application/json"}, + body=json.dumps( + {"module": "zeek_intel_export", "data": event([attribute("url", "http://example.org/%ff%00")])} + ), + ) + assert result.code == 200 + body = json.loads(result.body) + assert body["sources"][0]["indicator"] == "example.org/\\xff\x00" + assert b"example.org/\\xff\\x00\tIntel::URL" in base64.b64decode(body["data"])