|
| 1 | +import json |
| 2 | +from urllib.parse import quote, urlparse |
| 3 | + |
| 4 | +import requests |
| 5 | +from pymisp import MISPEvent |
| 6 | + |
| 7 | +from . import check_input_attribute, standard_error_message |
| 8 | + |
| 9 | +misperrors = {"error": "Error"} |
| 10 | +mispattributes = { |
| 11 | + "input": ["domain", "hostname", "url", "ip-src", "ip-dst", "domain|ip"], |
| 12 | + "output": ["domain", "hostname", "ip-src", "text", "link"], |
| 13 | + "format": "misp_standard", |
| 14 | +} |
| 15 | +moduleinfo = { |
| 16 | + "version": "1.0", |
| 17 | + "author": "Jonas Lejon", |
| 18 | + "description": "Query ScanMalware for subdomains, browser-observed hosts and sandbox scan results.", |
| 19 | + "module-type": ["expansion", "hover"], |
| 20 | + "name": "ScanMalware Lookup", |
| 21 | + "logo": "scanmalware.png", |
| 22 | + "requirements": [], |
| 23 | + "features": ( |
| 24 | + "The module takes a domain, hostname, URL or IP attribute and queries the public ScanMalware API." |
| 25 | + " For a domain or hostname it returns subdomains seen in Certificate Transparency DNS records, and the" |
| 26 | + " hosts a browser actually resolved and requested while rendering pages on that domain. For a URL it" |
| 27 | + " returns the most recent sandboxed scan of that host, with its verdict and a link to the full report." |
| 28 | + " For an IP it returns the URLs in the public scan archive that resolved to that address." |
| 29 | + "\n\nNo API key is required: the ScanMalware API is anonymous, and a key only raises the rate limit." |
| 30 | + " Only the queried indicator leaves the MISP instance." |
| 31 | + ), |
| 32 | + "references": ["https://scanmalware.com", "https://scanmalware.com/api-docs"], |
| 33 | + "input": "A domain, hostname, URL or IP address.", |
| 34 | + "output": "Domain attributes for the hosts found, plus text and link attributes describing the scan.", |
| 35 | +} |
| 36 | +# No API key: the public API is anonymous. api_url is exposed only so an operator can |
| 37 | +# point the module at a different deployment; max_results bounds how many attributes a |
| 38 | +# single enrichment may add. |
| 39 | +moduleconfig = ["api_url", "max_results"] |
| 40 | + |
| 41 | +DEFAULT_API_URL = "https://scanmalware.com" |
| 42 | +USER_AGENT = "scanmalware-misp/1.0 (+https://github.com/MISP/misp-modules)" |
| 43 | + |
| 44 | +# The API rejects a larger value rather than clamping it. |
| 45 | +SUBDOMAIN_LIMIT = 5000 |
| 46 | + |
| 47 | +# Cold queries have been measured at several seconds; a short timeout drops real results. |
| 48 | +TIMEOUT = 30 |
| 49 | + |
| 50 | +# A busy domain can have well over a thousand subdomains (cloudflare.com returns ~1200). |
| 51 | +# Adding that many attributes to one event is unusable in practice, so results are capped |
| 52 | +# and the cap is reported rather than applied silently. |
| 53 | +DEFAULT_MAX_RESULTS = 200 |
| 54 | + |
| 55 | + |
| 56 | +def _query_value(attribute): |
| 57 | + """Return the value to look up, and whether it is an IP.""" |
| 58 | + value = str(attribute.get("value", "")).strip() |
| 59 | + attribute_type = attribute.get("type") |
| 60 | + |
| 61 | + if attribute_type == "domain|ip" and "|" in value: |
| 62 | + return value.split("|", 1)[0], False |
| 63 | + if attribute_type in ("ip-src", "ip-dst"): |
| 64 | + return value, True |
| 65 | + if attribute_type == "url": |
| 66 | + return (urlparse(value).hostname or value), False |
| 67 | + return value, False |
| 68 | + |
| 69 | + |
| 70 | +def _get(api_url, path, params=None): |
| 71 | + """GET a JSON document, or None when the endpoint has nothing to give.""" |
| 72 | + try: |
| 73 | + response = requests.get( |
| 74 | + f"{api_url}{path}", |
| 75 | + params=params, |
| 76 | + headers={"User-Agent": USER_AGENT, "Accept": "application/json"}, |
| 77 | + timeout=TIMEOUT, |
| 78 | + ) |
| 79 | + if response.status_code == 404: |
| 80 | + return None |
| 81 | + response.raise_for_status() |
| 82 | + return response.json() |
| 83 | + except (requests.exceptions.RequestException, ValueError): |
| 84 | + return None |
| 85 | + |
| 86 | + |
| 87 | +class ScanMalwareParser: |
| 88 | + def __init__(self, api_url, max_results=DEFAULT_MAX_RESULTS): |
| 89 | + self.api_url = api_url |
| 90 | + self.max_results = max_results |
| 91 | + self.misp_event = MISPEvent() |
| 92 | + self.found = False |
| 93 | + self.added = 0 |
| 94 | + self.capped = False |
| 95 | + |
| 96 | + def _add(self, **kwargs): |
| 97 | + """Add one attribute, respecting the result cap.""" |
| 98 | + if self.added >= self.max_results: |
| 99 | + self.capped = True |
| 100 | + return False |
| 101 | + self.misp_event.add_attribute(**kwargs) |
| 102 | + self.added += 1 |
| 103 | + self.found = True |
| 104 | + return True |
| 105 | + |
| 106 | + def _add_hosts(self, hosts, comment): |
| 107 | + for host in sorted(set(hosts)): |
| 108 | + # Wildcard certificates appear literally as "*.example.com" and are not |
| 109 | + # resolvable hosts. |
| 110 | + if not host or host.startswith("*."): |
| 111 | + continue |
| 112 | + if not self._add(type="domain", value=host, comment=comment): |
| 113 | + return |
| 114 | + |
| 115 | + def parse_domain(self, domain): |
| 116 | + """Subdomains from Certificate Transparency, and hosts a browser contacted.""" |
| 117 | + ct = _get( |
| 118 | + self.api_url, |
| 119 | + f"/api/v1/ct/dns/{quote(domain, safe='')}", |
| 120 | + {"subdomain_limit": SUBDOMAIN_LIMIT}, |
| 121 | + ) |
| 122 | + if ct: |
| 123 | + self._add_hosts(ct.get("subdomains") or [], "ScanMalware: seen in Certificate Transparency DNS records") |
| 124 | + if ct.get("subdomains_truncated"): |
| 125 | + self._add( |
| 126 | + type="text", |
| 127 | + value=f"ScanMalware returned a truncated subdomain list for {domain}; more exist.", |
| 128 | + comment="ScanMalware: partial result", |
| 129 | + disable_correlation=True, |
| 130 | + ) |
| 131 | + |
| 132 | + # A different population: hosts a browser resolved and requested. Dev and |
| 133 | + # staging hosts holding no certificate appear only here. |
| 134 | + hosts = _get( |
| 135 | + self.api_url, |
| 136 | + f"/api/v1/hosts/{quote(domain, safe='')}", |
| 137 | + {"subdomains_only": "true"}, |
| 138 | + ) |
| 139 | + if hosts: |
| 140 | + self._add_hosts(hosts.get("subdomains") or [], "ScanMalware: contacted by a browser during a scan") |
| 141 | + |
| 142 | + def parse_url(self, hostname, url_value): |
| 143 | + """The most recent sandboxed scan of the host, with its verdict.""" |
| 144 | + scans = _get( |
| 145 | + self.api_url, |
| 146 | + f"/api/v1/domains/{quote(hostname, safe='')}/scans", |
| 147 | + {"limit": 5, "status": "completed"}, |
| 148 | + ) |
| 149 | + rows = scans if isinstance(scans, list) else (scans or {}).get("results") or [] |
| 150 | + if not rows: |
| 151 | + return |
| 152 | + |
| 153 | + scan = rows[0] |
| 154 | + scan_id = scan.get("scan_id") |
| 155 | + if not scan_id: |
| 156 | + return |
| 157 | + |
| 158 | + result = _get(self.api_url, f"/api/v1/result/{quote(scan_id, safe='')}") |
| 159 | + verdict = ((result or {}).get("security_verdict") or {}).get("verdict") |
| 160 | + summary = f"ScanMalware scan of {scan.get('url', url_value)}" |
| 161 | + if verdict: |
| 162 | + summary = f"{summary}: {verdict}" |
| 163 | + |
| 164 | + self._add(type="text", value=summary, comment="ScanMalware: sandbox verdict", disable_correlation=True) |
| 165 | + self._add( |
| 166 | + type="link", |
| 167 | + value=f"{self.api_url}/result/{scan_id}", |
| 168 | + comment="ScanMalware: full sandboxed report", |
| 169 | + disable_correlation=True, |
| 170 | + ) |
| 171 | + |
| 172 | + for row in (result or {}).get("ip_table") or []: |
| 173 | + if row.get("ip"): |
| 174 | + if not self._add( |
| 175 | + type="ip-src", value=row["ip"], comment="ScanMalware: contacted while rendering the page" |
| 176 | + ): |
| 177 | + break |
| 178 | + |
| 179 | + def parse_ip(self, ip_address): |
| 180 | + """URLs in the public scan archive that resolved to this address.""" |
| 181 | + search = _get(self.api_url, f"/api/v1/search/ip/{quote(ip_address, safe='')}", {"limit": 20}) |
| 182 | + results = (search or {}).get("results") or [] |
| 183 | + if not results: |
| 184 | + return |
| 185 | + |
| 186 | + self._add( |
| 187 | + type="text", |
| 188 | + value=f"ScanMalware: {search.get('total', len(results))} archived scan(s) resolved to {ip_address}", |
| 189 | + comment="ScanMalware: scan archive", |
| 190 | + disable_correlation=True, |
| 191 | + ) |
| 192 | + |
| 193 | + hosts = [] |
| 194 | + for row in results: |
| 195 | + hostname = urlparse(row.get("url") or "").hostname |
| 196 | + if hostname: |
| 197 | + hosts.append(hostname) |
| 198 | + self._add_hosts(hosts, "ScanMalware: scanned and resolved to this address") |
| 199 | + |
| 200 | + def get_results(self): |
| 201 | + if self.capped: |
| 202 | + # Report the cap instead of returning a silently shortened list. |
| 203 | + self.misp_event.add_attribute( |
| 204 | + type="text", |
| 205 | + value=( |
| 206 | + f"ScanMalware returned more results than the {self.max_results}-attribute limit;" |
| 207 | + " raise max_results in the module config to see the rest." |
| 208 | + ), |
| 209 | + comment="ScanMalware: partial result", |
| 210 | + disable_correlation=True, |
| 211 | + ) |
| 212 | + if not self.found: |
| 213 | + return {"error": "No ScanMalware results for this attribute."} |
| 214 | + event = json.loads(self.misp_event.to_json()) |
| 215 | + results = {key: event[key] for key in ("Attribute", "Object") if event.get(key)} |
| 216 | + if not results: |
| 217 | + return {"error": "No ScanMalware results for this attribute."} |
| 218 | + return {"results": results} |
| 219 | + |
| 220 | + |
| 221 | +def handler(q=False): |
| 222 | + if q is False: |
| 223 | + return False |
| 224 | + request = json.loads(q) |
| 225 | + |
| 226 | + if not request.get("attribute") or not check_input_attribute(request["attribute"]): |
| 227 | + return {"error": f"{standard_error_message}, which should contain at least a type, a value and an UUID."} |
| 228 | + |
| 229 | + attribute = request["attribute"] |
| 230 | + if attribute.get("type") not in mispattributes["input"]: |
| 231 | + return {"error": "Unsupported attribute type."} |
| 232 | + |
| 233 | + query, is_ip = _query_value(attribute) |
| 234 | + if not query: |
| 235 | + return {"error": "The provided attribute value is empty."} |
| 236 | + |
| 237 | + config = request.get("config") or {} |
| 238 | + api_url = str(config.get("api_url") or DEFAULT_API_URL).rstrip("/") |
| 239 | + |
| 240 | + try: |
| 241 | + max_results = int(config.get("max_results") or DEFAULT_MAX_RESULTS) |
| 242 | + except (TypeError, ValueError): |
| 243 | + max_results = DEFAULT_MAX_RESULTS |
| 244 | + |
| 245 | + parser = ScanMalwareParser(api_url, max_results) |
| 246 | + if is_ip: |
| 247 | + parser.parse_ip(query) |
| 248 | + elif attribute["type"] == "url": |
| 249 | + parser.parse_url(query, attribute.get("value", "")) |
| 250 | + else: |
| 251 | + parser.parse_domain(query) |
| 252 | + return parser.get_results() |
| 253 | + |
| 254 | + |
| 255 | +def introspection(): |
| 256 | + return mispattributes |
| 257 | + |
| 258 | + |
| 259 | +def version(): |
| 260 | + moduleinfo["config"] = moduleconfig |
| 261 | + return moduleinfo |
0 commit comments