diff --git a/documentation/expansion/shodan_internetdb.md b/documentation/expansion/shodan_internetdb.md new file mode 100644 index 000000000..73b9f0667 --- /dev/null +++ b/documentation/expansion/shodan_internetdb.md @@ -0,0 +1,37 @@ +# Shodan InternetDB + +Look up exposed services, hostnames, CPEs and vulnerabilities in the free Shodan InternetDB API. + +No API key is required. Supports IPv4 and IPv6. A 404 means no information is available. Hostnames and vulnerabilities are pivotable attributes; ports are grouped in an ip-port object. Service tags are retained as context, without automatically tagging the source as malicious. + +Input: ip-src, ip-dst. Output: An ip-port object and hostname, CPE and vulnerability attributes. + +Configuration: none. Required: none. +Requests use a 30 second timeout and verify TLS certificates. Errors distinguish +missing configuration, invalid input, authentication failures, rate limits and +invalid responses. No API credentials are included in error messages. + +All new enrichment attributes use `to_ids=false`. Source attributes retain their +UUID and metadata. Returned objects reference the source attribute where possible. +Empty lookups return the source attribute without fabricated enrichment. + +References: + +- https://internetdb.shodan.io/ + +Example request: + +```json +{ + "module": "shodan_internetdb", + "attribute": { + "type": "ip-dst", + "value": "1.1.1.1", + "uuid": "32876e4a-916d-4935-9e51-7b69a76718f8" + }, + "config": {} +} +``` + +Validation: `pytest tests/test_shodan_internetdb.py` uses mocked service responses; +no paid API credentials are needed. diff --git a/misp_modules/modules/expansion/shodan_internetdb.py b/misp_modules/modules/expansion/shodan_internetdb.py new file mode 100644 index 000000000..95269048f --- /dev/null +++ b/misp_modules/modules/expansion/shodan_internetdb.py @@ -0,0 +1,130 @@ +import json +from ipaddress import ip_address +from urllib.parse import quote + +import requests +from pymisp import MISPEvent, MISPObject + +misperrors = {"error": "Error"} +mispattributes = {"input": ["ip-src", "ip-dst"], "format": "misp_standard"} +moduleconfig = [] +moduleinfo = { + "version": "1", + "author": "MISP Project", + "name": "Shodan InternetDB", + "description": "Look up exposed services, hostnames, CPEs and vulnerabilities in the free Shodan InternetDB API.", + "module-type": ["expansion"], + "logo": "", + "requirements": ["requests"], + "features": ( + "No API key is required. Supports IPv4 and IPv6. A 404 means no information is available. Hostnames and" + " vulnerabilities are pivotable attributes; ports are grouped in an ip-port object. Service tags are retained" + " as context, without automatically tagging the source as malicious." + ), + "references": ["https://internetdb.shodan.io/"], + "input": "ip-src, ip-dst MISP attribute.", + "output": "An ip-port object and hostname, CPE and vulnerability attributes.", +} +TIMEOUT = 30 + + +class ModuleError(ValueError): + """An actionable module error safe to return to the caller.""" + + +def _get(url, **kwargs): + try: + response = requests.get(url, timeout=TIMEOUT, allow_redirects=False, **kwargs) + except requests.RequestException: + raise ModuleError("The service request failed or timed out.") from None + if response.status_code == 404: + return None + if response.status_code == 204: + return None + if response.status_code != 200: + raise ModuleError(f"The service returned HTTP {response.status_code}.") + try: + return response.json() + except ValueError: + raise ModuleError("The service returned invalid JSON.") from None + + +def _ip(value): + return str(ip_address(value)) + + +def _annotation(event, attribute, title, data): + obj = MISPObject("annotation") + obj.comment = title + obj.add_attribute("text", value=json.dumps(data, sort_keys=True, indent=2), to_ids=False, disable_correlation=True) + obj.add_reference(attribute["uuid"], "describes") + event.add_object(obj) + + +def _results(event): + data = json.loads(event.to_json()) + return {"results": {key: data.get(key, []) for key in ("Attribute", "Object")}} + + +def handler(q=False): + if q is False: + return False + try: + request = json.loads(q) + if not isinstance(request, dict): + raise ModuleError("The request must be a JSON object.") + attribute = request.get("attribute") + if not isinstance(attribute, dict) or not all((attribute.get(k) for k in ("type", "value", "uuid"))): + raise ModuleError("An attribute containing type, value and uuid is required.") + if attribute["type"] not in mispattributes["input"]: + raise ModuleError("Unsupported attribute type.") + if not isinstance(attribute["value"], str) or not attribute["value"].strip(): + raise ModuleError("The attribute value must be a nonempty string.") + config = request.get("config") or {} + if not isinstance(config, dict): + raise ModuleError("The config must be a JSON object.") + event = MISPEvent() + event.add_attribute(**attribute) + return _expand(event, attribute, config) + except ModuleError as exc: + return {"error": str(exc)} + except (ValueError, TypeError, KeyError, OverflowError, OSError): + return {"error": "Invalid request or service response."} + + +def introspection(): + return mispattributes + + +def version(): + return dict(moduleinfo, config=moduleconfig) + + +def _expand(event, attribute, config): + address = _ip(attribute["value"].strip()) + data = _get("https://internetdb.shodan.io/" + quote(address, safe="")) + if data is None: + return _results(event) + if not isinstance(data, dict) or _ip(data.get("ip", "")) != address: + raise ModuleError("InternetDB returned an unexpected IP address or response.") + for key in ("ports", "hostnames", "cpes", "vulns", "tags"): + if not isinstance(data.get(key, []), list): + raise ModuleError("InternetDB returned an unexpected response.") + if data.get("ports"): + obj = MISPObject("ip-port") + obj.add_attribute(attribute["type"], value=address, to_ids=False) + relation = attribute["type"].replace("ip-", "") + "-port" + for port in dict.fromkeys(data["ports"]): + if isinstance(port, bool) or not isinstance(port, int) or (not 0 <= port <= 65535): + raise ModuleError("InternetDB returned an invalid port.") + obj.add_attribute(relation, value=port, to_ids=False) + obj.add_reference(attribute["uuid"], "describes") + event.add_object(obj) + for key, kind in (("hostnames", "hostname"), ("cpes", "cpe"), ("vulns", "vulnerability")): + for value in dict.fromkeys(data.get(key, [])): + if not isinstance(value, str) or not value: + raise ModuleError("InternetDB returned an invalid value.") + event.add_attribute(kind, value, to_ids=False, comment="Shodan InternetDB") + if data.get("tags"): + _annotation(event, attribute, "Shodan InternetDB service tags", {"tags": data["tags"]}) + return _results(event) diff --git a/tests/test_shodan_internetdb.py b/tests/test_shodan_internetdb.py new file mode 100644 index 000000000..8e1b807ac --- /dev/null +++ b/tests/test_shodan_internetdb.py @@ -0,0 +1,152 @@ +import json +from unittest.mock import Mock, patch + +import pytest +import requests + +from misp_modules.modules.expansion import shodan_internetdb as module + +UUID = "32876e4a-916d-4935-9e51-7b69a76718f8" +SECRET = "test-api-secret" + + +def query(kind="ip-dst", value="1.1.1.1", config=None): + return json.dumps( + { + "attribute": {"type": kind, "value": value, "uuid": UUID, "to_ids": True, "comment": "source metadata"}, + "config": {} if config is None else config, + } + ) + + +def response(payload=None, status=200): + result = Mock(status_code=status, headers={}) + result.json.return_value = payload + result.iter_content.return_value = iter([b"sample bytes"]) + return result + + +def handle(payload, **kwargs): + with patch.object(module.requests, "get", return_value=response(payload)) as get: + result = module.handler(query(**kwargs)) + return result, get + + +def objects(result, name): + return [obj for obj in result["results"]["Object"] if obj["name"] == name] + + +def values(obj): + return {item["object_relation"]: item["value"] for item in obj["Attribute"]} + + +def assert_source(result): + source = next(item for item in result["results"]["Attribute"] if item["uuid"] == UUID) + assert source["to_ids"] is True + assert source["comment"] == "source metadata" + for obj in result["results"]["Object"]: + assert any(ref["referenced_uuid"] == UUID for ref in obj["ObjectReference"]) + assert all(item["to_ids"] is False for item in obj["Attribute"]) + assert all(item["to_ids"] is False for item in result["results"]["Attribute"] if item["uuid"] != UUID) + + +def test_discovery_contract(): + assert module.handler(False) is False + assert module.introspection()["format"] == "misp_standard" + assert module.version()["config"] == module.moduleconfig + assert "expansion" in module.version()["module-type"] + + +@pytest.mark.parametrize( + "raw", ["{", "[]", "null", "{}", '{"attribute": []}', '{"attribute":{"type":"ip-dst","value":"1.1.1.1"}}'] +) +def test_invalid_request_never_calls_service(raw): + with patch.object(module.requests, "get") as get: + assert "error" in module.handler(raw) + get.assert_not_called() + + +def test_unsupported_type_never_calls_service(): + with patch.object(module.requests, "get") as get: + assert "error" in module.handler(query(kind="comment")) + get.assert_not_called() + + +@pytest.mark.parametrize("value", ["", " ", 123, ["example.com"]]) +def test_invalid_values_never_call_service(value): + with patch.object(module.requests, "get") as get: + assert "error" in module.handler(query(value=value)) + get.assert_not_called() + + +@pytest.mark.parametrize("status", [401, 403, 429, 500, 302]) +def test_http_failures_are_reported(status): + with patch.object(module.requests, "get", return_value=response(status=status)): + result = module.handler(query()) + assert str(status) in result["error"] + assert SECRET not in result["error"] + + +@pytest.mark.parametrize("failure", [requests.Timeout, requests.ConnectionError]) +def test_network_failures_do_not_leak_credentials(failure): + with patch.object(module.requests, "get", side_effect=failure("url?apikey=" + SECRET)): + result = module.handler(query()) + assert "error" in result + assert SECRET not in result["error"] + + +@pytest.mark.parametrize("status", [404, 204]) +def test_empty_service_response_preserves_source(status): + with patch.object(module.requests, "get", return_value=response(status=status)): + result = module.handler(query()) + assert_source(result) + assert result["results"]["Object"] == [] + + +def test_invalid_service_json_is_reported(): + reply = response() + reply.json.side_effect = ValueError("invalid JSON " + SECRET) + with patch.object(module.requests, "get", return_value=reply): + result = module.handler(query()) + assert "error" in result + assert SECRET not in result["error"] + + +@pytest.mark.parametrize("payload", [[], "wrong", 1]) +def test_unexpected_service_shape_is_reported(payload): + result, _ = handle(payload) + assert "error" in result + + +PAYLOAD = { + "ip": "1.1.1.1", + "ports": [53, 443, 443], + "hostnames": ["one.one.one.one"], + "cpes": ["cpe:/a:example:server:1"], + "vulns": ["CVE-2025-1234"], + "tags": ["cdn"], +} + + +def test_host_intelligence_is_pivotable_and_referenced(): + result, get = handle(PAYLOAD) + assert_source(result) + obj = objects(result, "ip-port")[0] + assert values(obj)["ip-dst"] == "1.1.1.1" + assert [x["value"] for x in obj["Attribute"] if x["object_relation"] == "dst-port"] == ["53", "443"] + assert {x["type"] for x in result["results"]["Attribute"]} == {"ip-dst", "hostname", "cpe", "vulnerability"} + assert json.loads(values(objects(result, "annotation")[0])["text"])["tags"] == ["cdn"] + get.assert_called_once_with("https://internetdb.shodan.io/1.1.1.1", timeout=30, allow_redirects=False) + + +def test_ipv6_source_port_direction(): + payload = dict(PAYLOAD, ip="2001:4860:4860::8888") + result, get = handle(payload, kind="ip-src", value=payload["ip"]) + assert "src-port" in values(objects(result, "ip-port")[0]) + assert "%3A" in get.call_args.args[0] + + +@pytest.mark.parametrize("changes", [{"ip": "8.8.8.8"}, {"ports": "443"}, {"ports": [65536]}, {"ports": [True]}]) +def test_mismatched_or_invalid_results(changes): + result, _ = handle(dict(PAYLOAD, **changes)) + assert "error" in result