Skip to content

Commit 32dd244

Browse files
authored
Merge pull request #803 from jonaslejon/feat/scanmalware-expansion
Add ScanMalware expansion module
2 parents b090673 + 12989fd commit 32dd244

3 files changed

Lines changed: 425 additions & 0 deletions

File tree

36.7 KB
Loading
Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
import json
2+
from urllib.parse import quote, urlparse
3+
4+
import requests
5+
from pymisp import MISPEvent
6+
7+
from . import check_input_attribute, standard_error_message
8+
9+
misperrors = {"error": "Error"}
10+
mispattributes = {
11+
"input": ["domain", "hostname", "url", "ip-src", "ip-dst", "domain|ip"],
12+
"output": ["domain", "hostname", "ip-src", "text", "link"],
13+
"format": "misp_standard",
14+
}
15+
moduleinfo = {
16+
"version": "1.0",
17+
"author": "Jonas Lejon",
18+
"description": "Query ScanMalware for subdomains, browser-observed hosts and sandbox scan results.",
19+
"module-type": ["expansion", "hover"],
20+
"name": "ScanMalware Lookup",
21+
"logo": "scanmalware.png",
22+
"requirements": [],
23+
"features": (
24+
"The module takes a domain, hostname, URL or IP attribute and queries the public ScanMalware API."
25+
" For a domain or hostname it returns subdomains seen in Certificate Transparency DNS records, and the"
26+
" hosts a browser actually resolved and requested while rendering pages on that domain. For a URL it"
27+
" returns the most recent sandboxed scan of that host, with its verdict and a link to the full report."
28+
" For an IP it returns the URLs in the public scan archive that resolved to that address."
29+
"\n\nNo API key is required: the ScanMalware API is anonymous, and a key only raises the rate limit."
30+
" Only the queried indicator leaves the MISP instance."
31+
),
32+
"references": ["https://scanmalware.com", "https://scanmalware.com/api-docs"],
33+
"input": "A domain, hostname, URL or IP address.",
34+
"output": "Domain attributes for the hosts found, plus text and link attributes describing the scan.",
35+
}
36+
# No API key: the public API is anonymous. api_url is exposed only so an operator can
37+
# point the module at a different deployment; max_results bounds how many attributes a
38+
# single enrichment may add.
39+
moduleconfig = ["api_url", "max_results"]
40+
41+
DEFAULT_API_URL = "https://scanmalware.com"
42+
USER_AGENT = "scanmalware-misp/1.0 (+https://github.com/MISP/misp-modules)"
43+
44+
# The API rejects a larger value rather than clamping it.
45+
SUBDOMAIN_LIMIT = 5000
46+
47+
# Cold queries have been measured at several seconds; a short timeout drops real results.
48+
TIMEOUT = 30
49+
50+
# A busy domain can have well over a thousand subdomains (cloudflare.com returns ~1200).
51+
# Adding that many attributes to one event is unusable in practice, so results are capped
52+
# and the cap is reported rather than applied silently.
53+
DEFAULT_MAX_RESULTS = 200
54+
55+
56+
def _query_value(attribute):
57+
"""Return the value to look up, and whether it is an IP."""
58+
value = str(attribute.get("value", "")).strip()
59+
attribute_type = attribute.get("type")
60+
61+
if attribute_type == "domain|ip" and "|" in value:
62+
return value.split("|", 1)[0], False
63+
if attribute_type in ("ip-src", "ip-dst"):
64+
return value, True
65+
if attribute_type == "url":
66+
return (urlparse(value).hostname or value), False
67+
return value, False
68+
69+
70+
def _get(api_url, path, params=None):
71+
"""GET a JSON document, or None when the endpoint has nothing to give."""
72+
try:
73+
response = requests.get(
74+
f"{api_url}{path}",
75+
params=params,
76+
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
77+
timeout=TIMEOUT,
78+
)
79+
if response.status_code == 404:
80+
return None
81+
response.raise_for_status()
82+
return response.json()
83+
except (requests.exceptions.RequestException, ValueError):
84+
return None
85+
86+
87+
class ScanMalwareParser:
88+
def __init__(self, api_url, max_results=DEFAULT_MAX_RESULTS):
89+
self.api_url = api_url
90+
self.max_results = max_results
91+
self.misp_event = MISPEvent()
92+
self.found = False
93+
self.added = 0
94+
self.capped = False
95+
96+
def _add(self, **kwargs):
97+
"""Add one attribute, respecting the result cap."""
98+
if self.added >= self.max_results:
99+
self.capped = True
100+
return False
101+
self.misp_event.add_attribute(**kwargs)
102+
self.added += 1
103+
self.found = True
104+
return True
105+
106+
def _add_hosts(self, hosts, comment):
107+
for host in sorted(set(hosts)):
108+
# Wildcard certificates appear literally as "*.example.com" and are not
109+
# resolvable hosts.
110+
if not host or host.startswith("*."):
111+
continue
112+
if not self._add(type="domain", value=host, comment=comment):
113+
return
114+
115+
def parse_domain(self, domain):
116+
"""Subdomains from Certificate Transparency, and hosts a browser contacted."""
117+
ct = _get(
118+
self.api_url,
119+
f"/api/v1/ct/dns/{quote(domain, safe='')}",
120+
{"subdomain_limit": SUBDOMAIN_LIMIT},
121+
)
122+
if ct:
123+
self._add_hosts(ct.get("subdomains") or [], "ScanMalware: seen in Certificate Transparency DNS records")
124+
if ct.get("subdomains_truncated"):
125+
self._add(
126+
type="text",
127+
value=f"ScanMalware returned a truncated subdomain list for {domain}; more exist.",
128+
comment="ScanMalware: partial result",
129+
disable_correlation=True,
130+
)
131+
132+
# A different population: hosts a browser resolved and requested. Dev and
133+
# staging hosts holding no certificate appear only here.
134+
hosts = _get(
135+
self.api_url,
136+
f"/api/v1/hosts/{quote(domain, safe='')}",
137+
{"subdomains_only": "true"},
138+
)
139+
if hosts:
140+
self._add_hosts(hosts.get("subdomains") or [], "ScanMalware: contacted by a browser during a scan")
141+
142+
def parse_url(self, hostname, url_value):
143+
"""The most recent sandboxed scan of the host, with its verdict."""
144+
scans = _get(
145+
self.api_url,
146+
f"/api/v1/domains/{quote(hostname, safe='')}/scans",
147+
{"limit": 5, "status": "completed"},
148+
)
149+
rows = scans if isinstance(scans, list) else (scans or {}).get("results") or []
150+
if not rows:
151+
return
152+
153+
scan = rows[0]
154+
scan_id = scan.get("scan_id")
155+
if not scan_id:
156+
return
157+
158+
result = _get(self.api_url, f"/api/v1/result/{quote(scan_id, safe='')}")
159+
verdict = ((result or {}).get("security_verdict") or {}).get("verdict")
160+
summary = f"ScanMalware scan of {scan.get('url', url_value)}"
161+
if verdict:
162+
summary = f"{summary}: {verdict}"
163+
164+
self._add(type="text", value=summary, comment="ScanMalware: sandbox verdict", disable_correlation=True)
165+
self._add(
166+
type="link",
167+
value=f"{self.api_url}/result/{scan_id}",
168+
comment="ScanMalware: full sandboxed report",
169+
disable_correlation=True,
170+
)
171+
172+
for row in (result or {}).get("ip_table") or []:
173+
if row.get("ip"):
174+
if not self._add(
175+
type="ip-src", value=row["ip"], comment="ScanMalware: contacted while rendering the page"
176+
):
177+
break
178+
179+
def parse_ip(self, ip_address):
180+
"""URLs in the public scan archive that resolved to this address."""
181+
search = _get(self.api_url, f"/api/v1/search/ip/{quote(ip_address, safe='')}", {"limit": 20})
182+
results = (search or {}).get("results") or []
183+
if not results:
184+
return
185+
186+
self._add(
187+
type="text",
188+
value=f"ScanMalware: {search.get('total', len(results))} archived scan(s) resolved to {ip_address}",
189+
comment="ScanMalware: scan archive",
190+
disable_correlation=True,
191+
)
192+
193+
hosts = []
194+
for row in results:
195+
hostname = urlparse(row.get("url") or "").hostname
196+
if hostname:
197+
hosts.append(hostname)
198+
self._add_hosts(hosts, "ScanMalware: scanned and resolved to this address")
199+
200+
def get_results(self):
201+
if self.capped:
202+
# Report the cap instead of returning a silently shortened list.
203+
self.misp_event.add_attribute(
204+
type="text",
205+
value=(
206+
f"ScanMalware returned more results than the {self.max_results}-attribute limit;"
207+
" raise max_results in the module config to see the rest."
208+
),
209+
comment="ScanMalware: partial result",
210+
disable_correlation=True,
211+
)
212+
if not self.found:
213+
return {"error": "No ScanMalware results for this attribute."}
214+
event = json.loads(self.misp_event.to_json())
215+
results = {key: event[key] for key in ("Attribute", "Object") if event.get(key)}
216+
if not results:
217+
return {"error": "No ScanMalware results for this attribute."}
218+
return {"results": results}
219+
220+
221+
def handler(q=False):
222+
if q is False:
223+
return False
224+
request = json.loads(q)
225+
226+
if not request.get("attribute") or not check_input_attribute(request["attribute"]):
227+
return {"error": f"{standard_error_message}, which should contain at least a type, a value and an UUID."}
228+
229+
attribute = request["attribute"]
230+
if attribute.get("type") not in mispattributes["input"]:
231+
return {"error": "Unsupported attribute type."}
232+
233+
query, is_ip = _query_value(attribute)
234+
if not query:
235+
return {"error": "The provided attribute value is empty."}
236+
237+
config = request.get("config") or {}
238+
api_url = str(config.get("api_url") or DEFAULT_API_URL).rstrip("/")
239+
240+
try:
241+
max_results = int(config.get("max_results") or DEFAULT_MAX_RESULTS)
242+
except (TypeError, ValueError):
243+
max_results = DEFAULT_MAX_RESULTS
244+
245+
parser = ScanMalwareParser(api_url, max_results)
246+
if is_ip:
247+
parser.parse_ip(query)
248+
elif attribute["type"] == "url":
249+
parser.parse_url(query, attribute.get("value", ""))
250+
else:
251+
parser.parse_domain(query)
252+
return parser.get_results()
253+
254+
255+
def introspection():
256+
return mispattributes
257+
258+
259+
def version():
260+
moduleinfo["config"] = moduleconfig
261+
return moduleinfo

0 commit comments

Comments
 (0)