diff --git a/README.md b/README.md index 22e119698..5b92c77e2 100644 --- a/README.md +++ b/README.md @@ -235,6 +235,7 @@ For further Information see the [license file](https://misp.github.io/misp-modul * [Export to Sentinel or Defender](https://misp.github.io/misp-modules/action_mod/#export-to-sentinel-or-defender) - Export indicators to Microsoft Sentinel or Microsoft Defender. Requires an existing installation of MISP2Sentinel or MISP2Defender. * [Mattermost](https://misp.github.io/misp-modules/action_mod/#mattermost) - Simplistic module to send message to a Mattermost channel. * [Nextcloud talk](https://misp.github.io/misp-modules/action_mod/#nextcloud-talk) - Simplistic module to send a message to a Nextcloud talk conversation. +* [SentinelOne IOC Publish](https://misp.github.io/misp-modules/action_mod/#sentinelone-ioc-publish) - Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. * [Slack](https://misp.github.io/misp-modules/action_mod/#slack) - Simplistic module to send messages to a Slack channel. * [Test action](https://misp.github.io/misp-modules/action_mod/#test-action) - This module is merely a test, always returning true. Triggers on event publishing. diff --git a/documentation/README.md b/documentation/README.md index 7f3725b56..366ee18bf 100644 --- a/documentation/README.md +++ b/documentation/README.md @@ -4352,6 +4352,32 @@ Simplistic module to send a message to a Nextcloud talk conversation. ----- +#### [SentinelOne IOC Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py) + +Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py)] + +- **features**: +>Publishes IDS-marked hashes, domains, URLs and IP addresses from supplied events and objects. Requires account or site scope, uses stable external IDs, checks existing records, and supports expiry and dry runs. Existing records are not changed or deleted. Credentials and console URL come from the module server environment. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinelone_ioc_publish.md). + +- **config**: +>{'params': {'scope': {'type': 'string', 'description': 'Required: account or site'}, 'scope_id': {'type': 'string', 'description': 'Required: the single SentinelOne account or site ID'}, 'source': {'type': 'string', 'description': 'Stable source namespace, unique to this MISP publisher', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': '1-180 days; maximum 30 when publishing IP addresses', 'value': '14'}, 'severity': {'type': 'string', 'description': 'SentinelOne severity score, integer 0-7', 'value': '4'}, 'dry_run': {'type': 'string', 'description': 'true previews payloads without credentials or network access', 'value': 'false'}}, 'blocking': True, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>MISP events in standard, bare or legacy format, supplied in workflow data. + +- **output**: +>Per-indicator publication outcomes and counts; errors signal incomplete publication. + +- **references**: +>https://github.com/Sentinel-One/ai-siem/blob/main/plugins/s1-secops-skills/skills/mgmt-console-api/references/tags/Threat_Intelligence.md + +- **requirements**: +> - requests +> - SentinelOne Threat Intelligence.view and Threat Intelligence.manage permissions + +----- + #### [Slack](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/slack.py) Simplistic module to send messages to a Slack channel. diff --git a/documentation/action_mod/sentinelone_ioc_publish.md b/documentation/action_mod/sentinelone_ioc_publish.md new file mode 100644 index 000000000..875fc05c8 --- /dev/null +++ b/documentation/action_mod/sentinelone_ioc_publish.md @@ -0,0 +1,159 @@ +# SentinelOne IOC Publish + +`sentinelone_ioc_publish` is a **workflow action** that publishes selected MISP +indicators to SentinelOne's Threat Intelligence database. Publishing uses the +native v2.1 IOC API with explicit account or site scope. Threat Intelligence +publication does not automatically add indicators to an endpoint blocklist. +The module uses the existing `requests` dependency. Existing MISP `file` and +`domain-ip` objects suffice; no new object templates are required. + +## Set up + +1. Create a SentinelOne API token with **Threat Intelligence.view** and + **Threat Intelligence.manage** permissions for the intended account or site. +2. Set `SENTINELONE_URL` to your HTTPS console origin, for example + `https://your-console.sentinelone.net`, and set `SENTINELONE_API_TOKEN` in the + environment of the **misp-modules service**. Restart that service. The URL must + not include credentials, an API path, a query or a fragment. Workflow parameters + cannot override the console URL or token. +3. Enable **SentinelOne IOC Publish** and add it to an event or ad-hoc workflow + supplying the intended full event and object attributes. Configure a single + account or site below. The module does not fetch missing event data from MISP. +4. Preview with `dry_run=true`, then set `dry_run=false` to publish. Dry runs need + no credentials and make no network requests; they cannot verify API permissions + or identify existing records. + +Some SentinelOne deployments reject multi-account tokens with error `4030010` even +when requests contain explicit account/site filters. Use a service user limited to +the intended account or site and verify its Threat Intelligence permissions. + +| Parameter | Default | Meaning | +| --- | --- | --- | +| `scope` | Required | `account` or `site`. Tenant-wide and group publication are not supported. | +| `scope_id` | Required | A single account or site ID, supplied as an 18–20 digit string. | +| `source` | `misp-modules` | Stable publisher namespace. Use a distinct value per MISP publisher. Accepts 1–64 ASCII characters, starting with a letter/digit, using letters, digits, spaces or `._:/-`. | +| `expiration_days` | `14` | Integer 1–180. Any input containing an eligible IP indicator requires 30 or fewer days; split mixed inputs to use longer expiry for other types. Applies to newly created records only. | +| `severity` | `4` | Integer 0–7, applied to newly created records. | +| `dry_run` | `false` | Return scoped creation payloads without authentication, lookups or writes. | + +Booleans accept JSON booleans, integers 0/1 and strings `true`, `false`, `1`, `0`. +Empty optional parameters use the documented defaults. MISP displays defaults as +placeholders; the module also applies them when values are not supplied. + +## Input and supported types + +`data` accepts a standard event (`{"Event": {...}}`), a bare event, a legacy +envelope with `Attribute`/`Object` alongside `Event`, or a list of these events. +Only supplied, non-deleted attributes with `to_ids=true` are eligible. Deleted +events and objects are excluded. + +| MISP type | SentinelOne type | Conversion | +| --- | --- | --- | +| `md5`, `sha1`, `sha256` | `MD5`, `SHA1`, `SHA256` | Validate hexadecimal length and lowercase. Empty-file hashes are excluded because SentinelOne rejects them. | +| `domain`, `hostname` | `DNS` | IDNA encode, lowercase, remove a trailing DNS dot and validate DNS labels and the 253-character normalized name limit. | +| `url` | `URL` | Require an absolute URL with a host, valid port and no whitespace; preserve its exact value. | +| `ip-src`, `ip-dst` | `IPV4`, `IPV6` | Validate and canonicalize. CIDRs and scoped IPv6 are excluded. | +| `domain\|ip` | DNS plus IP | Publish two independent atomic indicators. | +| `filename\|md5`, `filename\|sha1`, `filename\|sha256`, `malware-sample` | File hash | Publish only the hash component, splitting at the last pipe. No filename or sample data is uploaded. | + +All indicators use the API's `EQUALS` matching method. Unsupported types, including +email addresses, filenames alone and port-qualified composites, are skipped with +counts. Port constraints are not discarded to create broader indicators. Invalid +supported values are counted as malformed and skipped; malformed input containers +and invalid configuration fail before network access. + +Canonical `(type, value)` pairs are deduplicated. All contributing event, attribute +and object references remain in each result's `sources` list. The remote description +contains event/attribute identifiers, limited to 500 characters. Free-text titles, +comments, tags and binary data are not copied. Select input according to the MISP +event's sharing policy and handling markings. + +The action advertises `support_filters=false`. Requests containing `filteredItems` +or `matchingData` fail to avoid publishing unintended unfiltered data. Select the +intended events upstream. + +## Identity and lifecycle + +A deterministic UUID `externalId` is derived from scope, scope ID, source, type and +canonical value. The same observable supplied by several MISP attributes therefore +has one publisher identity. Changing the source or scope creates a different +identity; keep those settings stable. + +Before creation, the module looks up the external ID within the configured scope. +An existing record is accepted only when its source, external ID, type, value, +matching method and exact scope agree, and its expiry is valid and in the future. +Incomplete lookups, multiple matches and mismatches fail without writing. A new +record is counted as created only after a matching scoped acknowledgement with a +remote UUID is received. + +The native IOC API exposes **no update endpoint**. Existing matching records are +reported as `existing` and left unchanged, including expiry, severity and source +description. Re-running does not renew their lifetime. Expired or invalid-expiry +records fail and require lifecycle administration in SentinelOne. The module does +not delete/recreate records. Deleted or non-IDS MISP attributes do not withdraw +remote indicators; use expiry or separate SentinelOne administration. + +Lookup and creation are separate operations. Concurrent invocations can race, and +the external ID is a correlation mechanism rather than a guarantee of exactly-once +creation. Writes are never automatically retried. Inspect SentinelOne before +retrying any request whose remote outcome is unknown. + +## API example + +POST this request to the misp-modules `/query` endpoint for a preview: + +```json +{ + "module": "sentinelone_ioc_publish", + "params": { + "scope": "account", + "scope_id": "225494730938493804", + "source": "MISP production", + "expiration_days": "14", + "dry_run": "true" + }, + "data": { + "Event": { + "uuid": "f1ef0409-c80a-4012-b401-b7d1050f4ed4", + "Attribute": [{ + "uuid": "aa09819b-f2d8-4661-83ea-d4897cafebc8", + "type": "ip-dst", + "value": "192.0.2.1", + "to_ids": true + }] + } + } +} +``` + +Responses contain `result_key=sentinelone_ioc_publish` and a `data` summary with +`success`, `dry_run`, `created`, `existing`, `failed`, `planned`, `skipped` and +`items`. Items include canonical type/value, external ID, source references and +status. Successful live items include the SentinelOne `uuid`; dry-run items include +the full scoped creation payload. + +Lookup/write failures set `success=false` and a top-level `error` with counts. +**Some indicators may already have been published.** API errors/warnings, malformed +responses and missing acknowledgements also fail, even on HTTP 2xx. Provider error +bodies and exception details are not returned, to avoid exposing credentials. + +This blocking action stops dependent workflow execution on failure. Successful +summaries are available under +`_env.misp_module_results.sentinelone_ioc_publish`. On failure, MISP may retain +only the top-level error; standalone API clients receive the per-item summary too. + +## Limits and validation + +Inputs may contain at most **100 unique indicators**; larger inputs fail before +network access. Split larger jobs. Each request has a 15-second timeout, and a +240-second processing budget is checked before each indicator and again after +lookup before creation. TLS verification is enabled, redirects are disabled, and +rate limits are reported without retries. Partial remote writes are not rolled +back if another indicator fails or the caller times out. + +Tests cover mocked API responses, actual `requests` serialization through mocked +HTTP transport, and the module server's discovery/query endpoints. They do not +change a live SentinelOne tenant. Validate permissions and tenant-specific behavior +in your deployment before enabling publication. + +API contract: [SentinelOne Threat Intelligence reference](https://github.com/Sentinel-One/ai-siem/blob/main/plugins/s1-secops-skills/skills/mgmt-console-api/references/tags/Threat_Intelligence.md). diff --git a/documentation/mkdocs/action_mod.md b/documentation/mkdocs/action_mod.md index 556486558..1c5762327 100644 --- a/documentation/mkdocs/action_mod.md +++ b/documentation/mkdocs/action_mod.md @@ -53,6 +53,32 @@ Simplistic module to send a message to a Nextcloud talk conversation. ----- +#### [SentinelOne IOC Publish](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py) + +Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. +[[source code](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/sentinelone_ioc_publish.py)] + +- **features**: +>Publishes IDS-marked hashes, domains, URLs and IP addresses from supplied events and objects. Requires account or site scope, uses stable external IDs, checks existing records, and supports expiry and dry runs. Existing records are not changed or deleted. Credentials and console URL come from the module server environment. See the [usage guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinelone_ioc_publish.md). + +- **config**: +>{'params': {'scope': {'type': 'string', 'description': 'Required: account or site'}, 'scope_id': {'type': 'string', 'description': 'Required: the single SentinelOne account or site ID'}, 'source': {'type': 'string', 'description': 'Stable source namespace, unique to this MISP publisher', 'value': 'misp-modules'}, 'expiration_days': {'type': 'string', 'description': '1-180 days; maximum 30 when publishing IP addresses', 'value': '14'}, 'severity': {'type': 'string', 'description': 'SentinelOne severity score, integer 0-7', 'value': '4'}, 'dry_run': {'type': 'string', 'description': 'true previews payloads without credentials or network access', 'value': 'false'}}, 'blocking': True, 'support_filters': False, 'expect_misp_core_format': True} + +- **input**: +>MISP events in standard, bare or legacy format, supplied in workflow data. + +- **output**: +>Per-indicator publication outcomes and counts; errors signal incomplete publication. + +- **references**: +>https://github.com/Sentinel-One/ai-siem/blob/main/plugins/s1-secops-skills/skills/mgmt-console-api/references/tags/Threat_Intelligence.md + +- **requirements**: +> - requests +> - SentinelOne Threat Intelligence.view and Threat Intelligence.manage permissions + +----- + #### [Slack](https://github.com/MISP/misp-modules/tree/main/misp_modules/modules/action_mod/slack.py) Simplistic module to send messages to a Slack channel. diff --git a/documentation/mkdocs/index.md b/documentation/mkdocs/index.md index 62b0bc3d2..5838fd7df 100644 --- a/documentation/mkdocs/index.md +++ b/documentation/mkdocs/index.md @@ -188,6 +188,7 @@ For more information: [Extending MISP with Python modules](https://www.misp-proj * [CrowdStrike IOC Publish](https://misp.github.io/misp-modules/action_mod/#crowdstrike-ioc-publish) - Create and renew module-owned Falcon custom IOCs from MISP event and object attributes. * [Mattermost](https://misp.github.io/misp-modules/action_mod/#mattermost) - Simplistic module to send message to a Mattermost channel. * [Nextcloud talk](https://misp.github.io/misp-modules/action_mod/#nextcloud-talk) - Simplistic module to send a message to a Nextcloud talk conversation. +* [SentinelOne IOC Publish](https://misp.github.io/misp-modules/action_mod/#sentinelone-ioc-publish) - Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope. * [Slack](https://misp.github.io/misp-modules/action_mod/#slack) - Simplistic module to send messages to a Slack channel. * [Test action](https://misp.github.io/misp-modules/action_mod/#test-action) - This module is merely a test, always returning true. Triggers on event publishing. diff --git a/misp_modules/modules/action_mod/sentinelone_ioc_publish.py b/misp_modules/modules/action_mod/sentinelone_ioc_publish.py new file mode 100644 index 000000000..e134e3903 --- /dev/null +++ b/misp_modules/modules/action_mod/sentinelone_ioc_publish.py @@ -0,0 +1,454 @@ +"""Publish MISP indicators to SentinelOne's scoped Threat Intelligence API.""" + +import ipaddress +import json +import os +import re +import time +import uuid +from collections import Counter +from datetime import datetime, timedelta, timezone +from urllib.parse import urlsplit + +import requests + +moduleinfo = { + "version": "1.0", + "author": "MISP Project", + "name": "SentinelOne IOC Publish", + "description": "Publish MISP event and object indicators to SentinelOne Threat Intelligence with explicit scope.", + "module-type": ["action"], + "logo": "", + "requirements": ["requests", "SentinelOne Threat Intelligence.view and Threat Intelligence.manage permissions"], + "features": ( + "Publishes IDS-marked hashes, domains, URLs and IP addresses from supplied events and objects. Requires account" + " or site scope, uses stable external IDs, checks existing records, and supports expiry and dry runs. Existing" + " records are not changed or deleted. Credentials and console URL come from the module server environment. See" + " the [usage" + " guide](https://github.com/MISP/misp-modules/blob/main/documentation/action_mod/sentinelone_ioc_publish.md)." + ), + "references": [ + "https://github.com/Sentinel-One/ai-siem/blob/main/plugins/s1-secops-skills/skills/mgmt-console-api/references/tags/Threat_Intelligence.md", + ], + "input": "MISP events in standard, bare or legacy format, supplied in workflow data.", + "output": "Per-indicator publication outcomes and counts; errors signal incomplete publication.", +} +moduleconfig = { + "params": { + "scope": {"type": "string", "description": "Required: account or site"}, + "scope_id": {"type": "string", "description": "Required: the single SentinelOne account or site ID"}, + "source": { + "type": "string", + "description": "Stable source namespace, unique to this MISP publisher", + "value": "misp-modules", + }, + "expiration_days": { + "type": "string", + "description": "1-180 days; maximum 30 when publishing IP addresses", + "value": "14", + }, + "severity": {"type": "string", "description": "SentinelOne severity score, integer 0-7", "value": "4"}, + "dry_run": { + "type": "string", + "description": "true previews payloads without credentials or network access", + "value": "false", + }, + }, + "blocking": True, + "support_filters": False, + "expect_misp_core_format": True, +} +returns = "data" +API_PATH = "/web/api/v2.1/threat-intelligence/iocs" +MAX_INDICATORS = 100 +DEADLINE_SECONDS = 240 +HASH_LENGTHS = {"md5": 32, "sha1": 40, "sha256": 64} +EMPTY_FILE_HASHES = { + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", +} +SUPPORTED_TYPES = ( + {"domain", "hostname", "url", "ip-src", "ip-dst", "domain|ip", "malware-sample"} + | HASH_LENGTHS.keys() + | {f"filename|{kind}" for kind in HASH_LENGTHS} +) + + +class PublishError(Exception): + """Sanitized failure suitable for a workflow result.""" + + +def _boolean(value): + if value is None or value == "": + return False + if isinstance(value, (bool, int)) and value in (0, 1): + return bool(value) + if isinstance(value, str) and value.lower() in ("true", "false", "1", "0"): + return value.lower() in ("true", "1") + raise ValueError("dry_run must be true, false, 1 or 0.") + + +def _integer(value, name, default, maximum): + if value is None or value == "": + return default + if isinstance(value, bool) or not re.fullmatch(r"[0-9]{1,3}", str(value)): + raise ValueError(f"{name} must be an integer.") + number = int(value) + if not (0 if name == "severity" else 1) <= number <= maximum: + raise ValueError(f"{name} is outside its supported range.") + return number + + +def _settings(params): + if not isinstance(params, dict): + raise ValueError("params must be an object.") + scope, scope_id = params.get("scope"), params.get("scope_id") + if scope not in ("account", "site"): + raise ValueError("scope must be account or site.") + if not isinstance(scope_id, str) or not re.fullmatch(r"[1-9][0-9]{17,19}", scope_id): + raise ValueError("scope_id must be a single 18-20 digit account or site ID.") + source = params.get("source") + if source is None or source == "": + source = "misp-modules" + if not isinstance(source, str) or not re.fullmatch(r"[a-zA-Z0-9][a-zA-Z0-9 ._:/-]{0,63}", source): + raise ValueError("source must be 1-64 ASCII letters, digits, spaces or ._:/- and start with a letter or digit.") + return { + "scope": scope, + "scope_id": scope_id, + "source": source, + "days": _integer(params.get("expiration_days"), "expiration_days", 14, 180), + "severity": _integer(params.get("severity"), "severity", 4, 7), + "dry_run": _boolean(params.get("dry_run")), + } + + +def _records(container, key): + records = container.get(key, []) + if not isinstance(records, list) or any(not isinstance(record, dict) for record in records): + raise ValueError(f"{key} must be a list of objects.") + return records + + +def _flag(value): + # MISP serializes boolean attributes as booleans, 0/1, or equivalent strings. + if value is None or value == "": + return False + if isinstance(value, (bool, int)) and value in (0, 1): + return bool(value) + if isinstance(value, str) and value.lower() in ("true", "false", "1", "0"): + return value.lower() in ("true", "1") + raise ValueError("Invalid MISP boolean flag.") + + +def _atomic(kind, value): + if kind in HASH_LENGTHS: + if not re.fullmatch(rf"[0-9a-fA-F]{{{HASH_LENGTHS[kind]}}}", value) or value.lower() == EMPTY_FILE_HASHES[kind]: + raise ValueError("Invalid file hash.") + return kind.upper(), value.lower() + if kind in ("ip-src", "ip-dst"): + if "%" in value: + raise ValueError("Scoped IPs are unsupported.") + address = ipaddress.ip_address(value) + return f"IPV{address.version}", str(address) + if kind == "url": + parsed = urlsplit(value) + if not parsed.scheme or not parsed.hostname or any(char.isspace() for char in value): + raise ValueError("URL must be absolute with a host and no whitespace.") + if parsed.port is not None and not 0 <= parsed.port <= 65535: + raise ValueError("Invalid URL port.") + return "URL", value + domain = value.removesuffix(".").encode("idna").decode("ascii").lower() + if len(domain) > 253 or any( + not re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) for label in domain.split(".") + ): + raise ValueError("Invalid DNS name.") + return "DNS", domain + + +def _indicators(kind, value): + if "|" not in kind and kind != "malware-sample": + return [_atomic(kind, value)] + first, separator, second = value.rpartition("|") + if not separator or not first.strip() or not second.strip(): + raise ValueError("Invalid composite.") + if kind == "domain|ip": + return [_atomic("domain", first), _atomic("ip-dst", second)] + return [_atomic("md5" if kind == "malware-sample" else kind.split("|")[1], second)] + + +def _collect(data, skipped): + if isinstance(data, dict): + data = [data] + if not isinstance(data, list) or any(not isinstance(event, dict) for event in data): + raise ValueError("data must contain an event or list of events.") + indicators = {} + for envelope in data: + event = envelope.get("Event", envelope) + if not isinstance(event, dict): + raise ValueError("Event must be an object.") + if _flag(event.get("deleted")): + continue + collections = dict(event, **envelope) + attributes = [(attribute, {}) for attribute in _records(collections, "Attribute")] + for obj in _records(collections, "Object"): + if not _flag(obj.get("deleted")): + attributes.extend((attribute, obj) for attribute in _records(obj, "Attribute")) + for attribute, obj in attributes: + if _flag(attribute.get("deleted")) or not _flag(attribute.get("to_ids")): + skipped["deleted or non-IDS"] += 1 + continue + kind, value = attribute.get("type"), attribute.get("value") + if not isinstance(kind, str) or kind not in SUPPORTED_TYPES: + skipped["unsupported"] += 1 + continue + try: + if not isinstance(value, str) or not value.strip(): + raise ValueError("Empty value.") + atoms = _indicators(kind, value) + except ValueError: + skipped["malformed"] += 1 + continue + ref = { + "event": str(event.get("uuid") or event.get("id") or "unknown"), + "attribute": str(attribute.get("uuid") or attribute.get("id") or "unknown"), + "misp_type": kind, + } + if obj: + ref["object"] = str(obj.get("uuid") or obj.get("id") or obj.get("name") or "unknown") + for atom in atoms: + refs = indicators.setdefault(atom, []) + if ref not in refs: + refs.append(ref) + if len(indicators) > MAX_INDICATORS: + raise ValueError(f"At most {MAX_INDICATORS} unique indicators are allowed; split the input.") + return indicators + + +def _endpoint(): + base = os.getenv("SENTINELONE_URL", "") + parsed = urlsplit(base) + if ( + parsed.scheme != "https" + or not parsed.hostname + or parsed.path not in ("", "/") + or parsed.query + or parsed.fragment + or parsed.username is not None + or parsed.password is not None + ): + raise ValueError( + "SENTINELONE_URL must be an HTTPS console origin without credentials, path, query or fragment." + ) + parsed.port # Validate an explicitly supplied port without changing the origin. + token = os.getenv("SENTINELONE_API_TOKEN", "") + if not token or any(ord(char) < 32 or ord(char) == 127 for char in token): + raise ValueError("SENTINELONE_API_TOKEN must be configured on the module server.") + return base.rstrip("/") + API_PATH, token + + +def _request(session, endpoint, method, **kwargs): + try: + response = session.request(method, endpoint, timeout=15, allow_redirects=False, verify=True, **kwargs) + except requests.RequestException: + raise PublishError( + f"{method} request failed; remote outcome may be unknown. Check SentinelOne before retrying." + ) from None + if not 200 <= response.status_code < 300: + raise PublishError(f"{method}: SentinelOne HTTP {response.status_code}.") + try: + body = response.json() + except ValueError: + raise PublishError(f"{method}: invalid JSON response; remote outcome may be unknown.") from None + if ( + not isinstance(body, dict) + or body.get("errors") + or body.get("warnings") + or not isinstance(body.get("data"), list) + ): + raise PublishError(f"{method}: invalid response or API errors/warnings; remote outcome may be unknown.") + return body + + +def _identity(record, item, settings): + scope_id = record.get("scopeId") + return ( + record.get("source") == item["source"] + and record.get("externalId") == item["externalId"] + and record.get("type") == item["type"] + and record.get("value") == item["value"] + and record.get("method") == "EQUALS" + and record.get("scope") == settings["scope"] + and scope_id in (settings["scope_id"], [settings["scope_id"]]) + and isinstance(record.get("uuid"), str) + and bool(record["uuid"]) + ) + + +def _publish(session, endpoint, item, settings, deadline): + scope_key = "accountIds" if settings["scope"] == "account" else "siteIds" + body = _request( + session, + endpoint, + "GET", + params={scope_key: settings["scope_id"], "tenant": "false", "externalId": item["externalId"], "limit": 2}, + ) + records = body["data"] + pagination = body.get("pagination") + if ( + not isinstance(pagination, dict) + or type(pagination.get("totalItems")) is not int + or pagination.get("nextCursor") + or pagination["totalItems"] != len(records) + or len(records) > 1 + ): + raise PublishError("GET: incomplete or ambiguous lookup; no IOC submitted.") + if records: + record = records[0] + if not isinstance(record, dict) or not _identity(record, item, settings): + raise PublishError("GET: existing record does not match publisher identity and scope; no IOC submitted.") + try: + expiry = datetime.fromisoformat(record["validUntil"].replace("Z", "+00:00")) + if expiry.tzinfo is None or expiry <= datetime.now(timezone.utc): + raise ValueError("Expired IOC.") + except (KeyError, TypeError, ValueError, AttributeError): + raise PublishError( + "Existing IOC expiry is invalid or elapsed; renew it in SentinelOne. No IOC submitted." + ) from None + return "existing", record["uuid"] + if time.monotonic() >= deadline: + raise PublishError("Publication time budget exceeded after lookup; no IOC submitted.") + body = _request( + session, endpoint, "POST", json={"filter": {scope_key: [settings["scope_id"]], "tenant": False}, "data": [item]} + ) + records = body["data"] + if len(records) != 1 or not isinstance(records[0], dict) or not _identity(records[0], item, settings): + raise PublishError("POST: no unambiguous scoped acknowledgement; remote outcome may be unknown.") + return "created", records[0]["uuid"] + + +def handler(q=False): + if not q: + return False + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ValueError("Request must be an object.") + if "filteredItems" in request or "matchingData" in request: + raise ValueError("Workflow filters/extracted data are unsupported; supply selected events in data.") + settings = _settings(request.get("params", {})) + skipped = Counter() + indicators = _collect(request.get("data"), skipped) + if settings["days"] > 30 and any(kind in ("IPV4", "IPV6") for kind, _ in indicators): + raise ValueError( + "SentinelOne IP indicators permit at most 30 expiration_days; split mixed inputs if needed." + ) + except (ValueError, TypeError) as exc: + return {"error": f"SentinelOne IOC publish: {exc}"} + summary = { + "success": True, + "dry_run": settings["dry_run"], + "created": 0, + "existing": 0, + "failed": 0, + "planned": 0, + "skipped": dict(skipped), + "items": [], + } + result = {"result_key": "sentinelone_ioc_publish", "data": summary} + if not indicators: + return result + session = None + if not settings["dry_run"]: + try: + endpoint, token = _endpoint() + except ValueError: + return { + "error": ( + "SentinelOne IOC publish: configure SENTINELONE_URL as a credential-free HTTPS console origin and" + " SENTINELONE_API_TOKEN on the module server." + ) + } + session = requests.Session() + session.headers.update( + {"Authorization": f"ApiToken {token}", "Accept": "application/json", "Content-Type": "application/json"} + ) + now = datetime.now(timezone.utc) + expiry = (now + timedelta(days=settings["days"])).isoformat(timespec="seconds").replace("+00:00", "Z") + deadline = time.monotonic() + DEADLINE_SECONDS + try: + for (kind, value), refs in indicators.items(): + external_id = str( + uuid.uuid5( + uuid.NAMESPACE_URL, + json.dumps( + [ + "misp-sentinelone-ioc-publish", + settings["scope"], + settings["scope_id"], + settings["source"], + kind, + value, + ] + ), + ) + ) + item = { + "type": kind, + "value": value, + "source": settings["source"], + "externalId": external_id, + "method": "EQUALS", + "validUntil": expiry, + "severity": settings["severity"], + "name": f"MISP {kind} {external_id}", + "description": ("MISP sources: " + "; ".join(f"{ref['event']}/{ref['attribute']}" for ref in refs))[ + :500 + ], + } + outcome = {"type": kind, "value": value, "externalId": external_id, "sources": refs} + if settings["dry_run"]: + scope_key = "accountIds" if settings["scope"] == "account" else "siteIds" + outcome.update( + status="planned", + payload={"filter": {scope_key: [settings["scope_id"]], "tenant": False}, "data": [item]}, + ) + summary["planned"] += 1 + else: + try: + if time.monotonic() >= deadline: + raise PublishError("Publication time budget exceeded; indicator not attempted.") + status, identifier = _publish(session, endpoint, item, settings, deadline) + outcome.update(status=status, uuid=identifier) + summary[status] += 1 + except PublishError as exc: + outcome.update(status="failed", error=str(exc)) + summary["failed"] += 1 + except Exception: + outcome.update( + status="failed", + error=( + "SentinelOne request failed; remote outcome may be unknown. Check SentinelOne before" + " retrying." + ), + ) + summary["failed"] += 1 + summary["items"].append(outcome) + finally: + if session is not None: + session.close() + if summary["failed"]: + summary["success"] = False + result["error"] = ( + f"SentinelOne IOC publish: {summary['failed']} failed; {summary['created']} created and" + f" {summary['existing']} existing unchanged. Publication may be partial." + ) + return result + + +def introspection(): + return {"config": moduleconfig} + + +def version(): + return dict(moduleinfo, config=moduleconfig) diff --git a/tests/test_sentinelone_ioc_publish.py b/tests/test_sentinelone_ioc_publish.py new file mode 100644 index 000000000..33d62b91f --- /dev/null +++ b/tests/test_sentinelone_ioc_publish.py @@ -0,0 +1,431 @@ +import copy +import json +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock +from urllib.parse import parse_qs, urlsplit + +import pytest +import requests +from pymisp import MISPObject +from tornado.testing import AsyncHTTPTestCase +from tornado.web import Application + +from misp_modules import __main__ as service +from misp_modules.modules.action_mod import sentinelone_ioc_publish as publish + +SCOPE_ID = "225494730938493804" + + +def attribute(kind="domain", value="example.org", **metadata): + return {"type": kind, "value": value, "to_ids": True, "uuid": "attribute-1", **metadata} + + +def event(attributes=None, objects=None, **metadata): + return {"uuid": "event-1", "Attribute": attributes or [], "Object": objects or [], **metadata} + + +def run(data=None, **params): + return publish.handler( + json.dumps( + { + "data": event([attribute()]) if data is None else data, + "params": {"scope": "account", "scope_id": SCOPE_ID, **params}, + } + ) + ) + + +def response(records=None, status=200, **body): + records = [] if records is None else records + result = MagicMock(spec=requests.Response) + result.status_code = status + result.json.return_value = { + "data": records, + "errors": None, + "pagination": {"totalItems": len(records), "nextCursor": None}, + **body, + } + return result + + +def existing(**metadata): + item = run(dry_run=True)["data"]["items"][0]["payload"]["data"][0] + return {**item, "uuid": "ioc-1", "scope": "account", "scopeId": SCOPE_ID, **metadata} + + +@pytest.fixture +def client(monkeypatch): + session = MagicMock(spec=requests.Session) + session.headers = {} + + def request(method, endpoint, **kwargs): + if method == "GET": + return response() + body = kwargs["json"] + scope = "account" if "accountIds" in body["filter"] else "site" + return response([{**body["data"][0], "uuid": "ioc-1", "scope": scope, "scopeId": SCOPE_ID}]) + + session.request.side_effect = request + constructor = MagicMock(return_value=session) + monkeypatch.setattr(publish.requests, "Session", constructor) + monkeypatch.setenv("SENTINELONE_URL", "https://console.example") + monkeypatch.setenv("SENTINELONE_API_TOKEN", "test-token") + session.constructor = constructor + return session + + +@pytest.mark.parametrize("scope,key", [("account", "accountIds"), ("site", "siteIds")]) +def test_publish_uses_scoped_v21_contract_and_checked_acknowledgement(client, scope, key): + before = datetime.now(timezone.utc) + result = run(scope=scope) + assert result["data"]["created"] == 1 and result["data"]["success"] is True + assert result["result_key"] == "sentinelone_ioc_publish" + assert client.headers["Authorization"] == "ApiToken test-token" + lookup, create = client.request.call_args_list + assert lookup.args == ("GET", "https://console.example" + publish.API_PATH) + assert lookup.kwargs["params"][key] == SCOPE_ID + assert lookup.kwargs["params"]["tenant"] == "false" + assert create.args[0] == "POST" + assert create.kwargs["allow_redirects"] is False and create.kwargs["verify"] is True + assert create.kwargs["timeout"] == 15 + body = create.kwargs["json"] + assert body["filter"] == {key: [SCOPE_ID], "tenant": False} + item = body["data"][0] + assert item["type"] == "DNS" and item["method"] == "EQUALS" + assert item["severity"] == 4 and item["externalId"] == lookup.kwargs["params"]["externalId"] + expiry = datetime.fromisoformat(item["validUntil"].replace("Z", "+00:00")) + assert before + timedelta(days=14, seconds=-1) <= expiry <= datetime.now(timezone.utc) + timedelta(days=14) + client.close.assert_called_once() + + +@pytest.mark.parametrize("scope_id", [SCOPE_ID, [SCOPE_ID]]) +def test_existing_owned_record_is_reported_without_mutation(client, scope_id): + client.request.side_effect = lambda *args, **kwargs: response([existing(scopeId=scope_id)]) + result = run() + assert result["data"]["existing"] == 1 and result["data"]["created"] == 0 + assert client.request.call_count == 1 and client.request.call_args.args[0] == "GET" + + +@pytest.mark.parametrize( + "metadata", + [ + {"source": "foreign"}, + {"externalId": "foreign"}, + {"type": "IPV4"}, + {"value": "other.example"}, + {"method": "CONTAINS"}, + {"scope": "global"}, + {"scopeId": "225494730938493805"}, + {"scopeId": [SCOPE_ID, "225494730938493805"]}, + {"uuid": ""}, + {"validUntil": "2020-01-01T00:00:00Z"}, + {"validUntil": None}, + {"validUntil": "not-a-date"}, + ], +) +def test_foreign_mismatched_or_expired_records_are_not_changed(client, metadata): + client.request.side_effect = lambda *args, **kwargs: response([existing(**metadata)]) + result = run() + assert result["data"]["failed"] == 1 and "error" in result + assert client.request.call_count == 1 + client.close.assert_called_once() + + +@pytest.mark.parametrize( + "body", + [ + response([{}], pagination={"totalItems": 2}), + response([], pagination={"totalItems": 1}), + response([], pagination={"totalItems": 0, "nextCursor": "more"}), + response([], pagination={"totalItems": False}), + response([], pagination=None), + response([{}, {}]), + ], +) +def test_lookup_must_be_complete_and_unambiguous_before_writing(client, body): + client.request.side_effect = None + client.request.return_value = body + assert run()["data"]["failed"] == 1 + assert client.request.call_count == 1 + + +@pytest.mark.parametrize( + "kind,value,expected", + [ + ("domain", "EXAMPLE.org.", [("DNS", "example.org")]), + ("hostname", "bücher.example", [("DNS", "xn--bcher-kva.example")]), + ("url", "https://example.org:8443/a?b=c", [("URL", "https://example.org:8443/a?b=c")]), + ("md5", "A" * 32, [("MD5", "a" * 32)]), + ("sha1", "A" * 40, [("SHA1", "a" * 40)]), + ("sha256", "A" * 64, [("SHA256", "a" * 64)]), + ("ip-src", "192.0.2.1", [("IPV4", "192.0.2.1")]), + ("ip-dst", "2001:0DB8:0000::1", [("IPV6", "2001:db8::1")]), + ("domain|ip", "example.org|192.0.2.1", [("DNS", "example.org"), ("IPV4", "192.0.2.1")]), + ("filename|md5", "a|b.exe|" + "a" * 32, [("MD5", "a" * 32)]), + ("filename|sha1", "a.exe|" + "a" * 40, [("SHA1", "a" * 40)]), + ("filename|sha256", "a.exe|" + "a" * 64, [("SHA256", "a" * 64)]), + ("malware-sample", "a.exe|" + "a" * 32, [("MD5", "a" * 32)]), + ], +) +def test_supported_types_are_normalized(client, kind, value, expected): + result = run(event([attribute(kind, value)])) + assert [(item["type"], item["value"]) for item in result["data"]["items"]] == expected + assert result["data"]["created"] == len(expected) + + +def test_deduplication_provenance_and_legacy_object_shapes(client): + obj = {"uuid": "object-1", "name": "domain-ip", "Attribute": [attribute("ip-dst", "192.0.2.1")]} + data = [ + {"Event": event([attribute(), attribute(uuid="attribute-2")], [obj])}, + {"Event": {"uuid": "event-2"}, "Attribute": [attribute()]}, + ] + original = copy.deepcopy(data) + result = run(data) + assert result["data"]["created"] == 2 + assert len(result["data"]["items"][0]["sources"]) == 3 + assert result["data"]["items"][1]["sources"][0]["object"] == "object-1" + assert data == original + + +def test_external_ids_are_stable_for_observable_and_different_across_scope_and_source(): + first = run(dry_run=True)["data"]["items"][0]["externalId"] + assert run(event([attribute(uuid="different")]), dry_run=True)["data"]["items"][0]["externalId"] == first + assert run(dry_run=True, scope="site")["data"]["items"][0]["externalId"] != first + assert run(dry_run=True, source="other")["data"]["items"][0]["externalId"] != first + + +@pytest.mark.parametrize( + "template,relation,value", [("file", "sha1", "a" * 40), ("domain-ip", "domain", "example.org")] +) +def test_existing_misp_object_templates_suffice(client, template, relation, value): + obj = MISPObject(template, strict=True) + obj.add_attribute(relation, value=value, to_ids=True) + obj._validate() + result = run(event(objects=[json.loads(obj.to_json())])) + assert result["data"]["created"] == 1 + assert result["data"]["items"][0]["sources"][0]["object"] == obj.uuid + + +def test_dry_run_has_no_credentials_or_network(client, monkeypatch): + monkeypatch.delenv("SENTINELONE_URL") + monkeypatch.delenv("SENTINELONE_API_TOKEN") + result = run(dry_run="true") + assert result["data"]["planned"] == 1 + client.constructor.assert_not_called() + assert "Authorization" not in json.dumps(result) + + +def test_deleted_context_and_unsupported_port_constraints_are_excluded(client): + data = [ + event( + [attribute(to_ids="false"), attribute(deleted="1"), attribute("ip-dst|port", "192.0.2.1|443")], + [{"deleted": True, "Attribute": [attribute()]}], + ), + event([attribute()], deleted=True), + ] + result = run(data) + assert result["data"]["skipped"] == {"deleted or non-IDS": 2, "unsupported": 1} + client.constructor.assert_not_called() + + +@pytest.mark.parametrize( + "kind,value", + [ + ("domain", "bad'name.example"), + ("domain", "a..example"), + ("ip-src", "192.0.2.0/24"), + ("ip-dst", "fe80::1%eth0"), + ("domain|ip", "example.org|not-ip"), + ("md5", "bad"), + ("sha1", "g" * 40), + ("url", "example.org"), + ("url", "https://example.org:bad/"), + ("domain", None), + ("domain", ""), + *[(kind, value) for kind, value in publish.EMPTY_FILE_HASHES.items()], + ], +) +def test_malformed_or_provider_rejected_values_are_skipped_without_network(client, kind, value): + assert run(event([attribute(kind, value)]))["data"]["skipped"] == {"malformed": 1} + client.constructor.assert_not_called() + + +@pytest.mark.parametrize( + "params", + [ + {"scope": "global"}, + {"scope": None}, + {"scope_id": "123"}, + {"scope_id": int(SCOPE_ID)}, + {"scope_id": SCOPE_ID + "," + SCOPE_ID}, + {"source": "bad'source"}, + {"source": False}, + {"severity": 8}, + {"severity": True}, + {"expiration_days": 181}, + {"expiration_days": 0}, + {"expiration_days": 1.5}, + {"dry_run": "maybe"}, + ], +) +def test_settings_validation_precedes_network(client, params): + assert "error" in run(**params) + client.constructor.assert_not_called() + + +def test_expiry_respects_provider_type_limits_before_network(client): + assert "error" in run(event([attribute("ip-dst", "192.0.2.1")]), expiration_days=31) + client.constructor.assert_not_called() + assert run(event([attribute("ip-dst", "192.0.2.1")]), expiration_days=30)["data"]["created"] == 1 + assert run(expiration_days=180)["data"]["created"] == 1 + + +@pytest.mark.parametrize( + "url", + [ + "http://console.example", + "https://:password@console.example", + "https://@console.example", + "https://console.example/path", + "https://console.example/?q=x", + "https://console.example/#x", + "https://console.example:bad", + "", + ], +) +def test_bad_console_origins_do_not_send_tokens_or_echo_credentials(client, monkeypatch, url): + monkeypatch.setenv("SENTINELONE_URL", url) + result = run() + assert "error" in result and "password" not in json.dumps(result) + client.constructor.assert_not_called() + + +def test_missing_credentials_and_malformed_requests_fail_without_network(client, monkeypatch): + monkeypatch.delenv("SENTINELONE_API_TOKEN") + assert "error" in run() + for payload in ( + [], + {"params": []}, + {"data": None}, + {"data": {"Event": "bad"}}, + {"data": event([attribute()]), "filteredItems": []}, + ): + base = {"params": {"scope": "account", "scope_id": SCOPE_ID}} + assert "error" in publish.handler(json.dumps({**base, **payload} if isinstance(payload, dict) else payload)) + assert publish.handler() is False + assert "error" in publish.handler("invalid JSON") + client.constructor.assert_not_called() + + +def test_limit_is_checked_before_network(client): + assert "error" in run(event([attribute(value=f"host{index}.example") for index in range(101)])) + client.constructor.assert_not_called() + + +@pytest.mark.parametrize("status", [301, 302, 401, 403, 429, 500]) +def test_http_errors_or_redirects_never_write_or_expose_provider_errors(client, status): + client.request.side_effect = None + client.request.return_value = response(status=status, errors=[{"detail": "test-token"}]) + result = run() + assert result["data"]["failed"] == 1 and "error" in result + assert "test-token" not in json.dumps(result) + assert client.request.call_count == 1 + client.close.assert_called_once() + + +@pytest.mark.parametrize( + "bad", [response(errors=[{}]), response(warnings=[{}]), response(records={}), response(pagination=None)] +) +def test_malformed_success_responses_are_not_treated_as_empty_lookups(client, bad): + client.request.side_effect = None + client.request.return_value = bad + assert run()["data"]["failed"] == 1 + assert client.request.call_count == 1 + + +def test_unacknowledged_write_reports_unknown_outcome_without_retry(client): + client.request.side_effect = [response(), response([{}])] + result = run() + assert result["data"]["failed"] == 1 and result["data"]["created"] == 0 + assert "unknown" in result["data"]["items"][0]["error"] + assert client.request.call_count == 2 + + +def test_partial_success_and_timeout_preserve_outcomes_without_echoing_secrets(client): + good = {**existing(), "uuid": "created-1"} + client.request.side_effect = [response(), response([good]), requests.Timeout("test-token")] + result = run(event([attribute(), attribute(value="other.example")])) + assert result["data"]["created"] == 1 and result["data"]["failed"] == 1 + assert result["data"]["success"] is False and "error" in result + assert "test-token" not in json.dumps(result) + + +def test_time_budget_is_rechecked_after_lookup(client, monkeypatch): + values = iter([0, 0, publish.DEADLINE_SECONDS + 1]) + monkeypatch.setattr(publish.time, "monotonic", lambda: next(values)) + assert run()["data"]["failed"] == 1 + assert client.request.call_count == 1 + + +def test_real_requests_transport_prepares_expected_wire_contract(monkeypatch): + calls = [] + + def send(self, request, **kwargs): + calls.append((request, kwargs)) + if request.method == "GET": + data = {"data": [], "pagination": {"totalItems": 0, "nextCursor": None}} + assert parse_qs(urlsplit(request.url).query)["siteIds"] == [SCOPE_ID] + else: + payload = json.loads(request.body) + assert payload["filter"] == {"siteIds": [SCOPE_ID], "tenant": False} + data = {"data": [{**payload["data"][0], "uuid": "ioc-1", "scope": "site", "scopeId": SCOPE_ID}]} + result = requests.Response() + result.status_code = 200 + result.headers["Content-Type"] = "application/json" + result._content = json.dumps(data).encode() + result.request = request + result.url = request.url + return result + + monkeypatch.setenv("SENTINELONE_URL", "https://console.example") + monkeypatch.setenv("SENTINELONE_API_TOKEN", "test-token") + monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send) + assert run(scope="site")["data"]["created"] == 1 + assert [request.method for request, _ in calls] == ["GET", "POST"] + assert all(request.headers["Authorization"] == "ApiToken test-token" for request, _ in calls) + + +class TestSentinelOnePublishHTTP(AsyncHTTPTestCase): + def get_app(self): + self.previous = service.MODULES_HANDLERS.copy() + self.cache = service.ListModules.CACHE + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update( + {"sentinelone_ioc_publish": publish, "type:sentinelone_ioc_publish": "action_mod"} + ) + service.ListModules.CACHE = None + return Application([(r"/modules", service.ListModules), (r"/query", service.QueryModule)]) + + def tearDown(self): + service.MODULES_HANDLERS.clear() + service.MODULES_HANDLERS.update(self.previous) + service.ListModules.CACHE = self.cache + super().tearDown() + + def test_discovery_and_dry_run(self): + descriptor = json.loads(self.fetch("/modules").body)[0] + assert descriptor["name"] == "sentinelone_ioc_publish" + assert descriptor["meta"]["config"]["blocking"] is True + result = self.fetch( + "/query", + method="POST", + headers={"Content-Type": "application/json"}, + body=json.dumps( + { + "module": "sentinelone_ioc_publish", + "data": event([attribute()]), + "params": {"scope": "site", "scope_id": SCOPE_ID, "dry_run": True}, + } + ), + ) + assert result.code == 200 and json.loads(result.body)["data"]["planned"] == 1