Merge branch 'master' into 60-new-vuln-drivers-definition-file

.github/ISSUE_TEMPLATE/bug_report.md

@@ -7,28 +7,34 @@ assignees: ''
 
 ---
 
-**Describe the bug**
+## Describe the bug
 A clear and concise description of what the bug is.
 
-**What side of Surveyor is impacted?**
+## What side of Surveyor is impacted?
 - [ ] Definition File
 - [ ] Code/Logic
+- [ ] Other (please explain)
 
-**What product is impacted?**
+## What product is impacted?
+- [ ] All Products
 - [ ] Carbon Black Response
 - [ ] Carbon Black Threat Hunter
+- [ ] Defender for Endpoints
+- [ ] SentinelOne
+- [ ] Cortex
 - [ ] Other
 
-**To Reproduce**
-**What did you do?**
+## Steps to reproduce
+
+### What did you do?
 What is the command line you're running that is causing the error? 
 Command line '...'
 
-**Expected behavior**
+### Expected behavior
 A clear and concise description of what you expected to happen.
 
-**Screenshots**
+### Screenshots
 If applicable, add screenshots to help explain your problem.
 
-**Additional context**
+### Additional context
 Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,23 +1,30 @@
 ---
 name: Feature request
 about: Suggest an idea for this project
-title: ''
+title: '[FR]'
 labels: feature
 assignees: ''
 
 ---
 
-**Which category is the feature part of?**
-- [ ] Carbon Black Response
-- [ ] Threat Hunter
+## Which category is the feature part of?
 - [ ] Definition File
-- [ ] Code Feature - impacts all products 
-- [ ] New product support
+- [ ] Code/Logic Feature 
+- [ ] Other (please explain)
+
+## Which product is the feature part of?
+- [ ] All Products
+- [ ] Carbon Black Response
+- [ ] Carbon Black Threat Hunter
+- [ ] Defender for Endpoints
+- [ ] SentinelOne
+- [ ] Cortex
+- [ ] Other
 
-**Use Cases**
+## Use Cases
 
-**Proposal**
+## Proposal
 A clear and concise description of what you want to happen.
 
-**Additional context**
+## Additional Context
 Add any other context or screenshots about the feature request here.

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "pip" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/test.yml

@@ -0,0 +1,43 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Test
+
+on:
+  push:
+    branches: [ "*" ]
+  pull_request:
+    branches: [ "*" ]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest pytest-mock mock mypy
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f requirements_sigma.txt ]; then pip install -r requirements_sigma.txt; fi
+        if [ -f requirements.types.txt ]; then pip install -r requirements.types.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest -vv
+    - name: Type check with mypy
+      run: |
+        mypy ./ --exclude 'setup\.py' --exclude 'tests/'

.gitignore

@@ -118,4 +118,5 @@ ENV/
 .idea/
 
 # other
-logs/
+logs/
+products/.DS_Store

README.md

@@ -4,12 +4,12 @@ Surveyor is a Python utility that queries Endpoint Detection and Response (EDR)
 products and summarizes the results. Security and IT teams can use Surveyor to
 baseline their environments and identify abnormal activity.
 
-## Current Version: 2.0
+## Current Version: 2.5.0
 
 Version 2.0 introduced breaking changes to the command line interface and support for SentinelOne. 
-If you are looking for the prior version of Surveyor, see the v1.0 release. 
+If you are looking for the prior version of Surveyor, see [past releases](https://github.com/redcanaryco/surveyor/releases).
 
-If you are new to version 2.0 please see the [Getting started](https://github.com/redcanaryco/surveyor/wiki/Getting-started) page of the wiki
+If you are new to version 2.X please see the [Getting started](https://github.com/redcanaryco/surveyor/wiki/Getting-started) page of the wiki
 and explore the new command line interface via `surveyor.py --help`.
 
 ## Analyze your endpoints
@@ -20,10 +20,11 @@ within an enterprise, who is using them, and how.
 
 Surveyor currently supports the following EDR platforms:
 
-- VMware Carbon Black® (Cb) Enterprise Response
-- VMware Carbon Black Cloud Enterprise EDR (formerly Carbon Black ThreatHunter)
+- Cortex XDR
 - Microsoft Defender for Endpoint
 - SentinelOne
+- VMware Carbon Black EDR (formerly Carbon Black Response)
+- VMware Carbon Black Cloud Enterprise EDR (formerly Carbon Black Cloud Threat Hunter)
 
 You can find out more about Surveyor from [this blog post](https://redcanary.com/blog/carbon-black-response-how-tos-surveyor/).
 
@@ -42,12 +43,18 @@ page of the wiki.
 
 #### Running the `sysinternals` definition file using the `cbr` product:
 
-```
+```bash
 surveyor.py --deffile sysinternals cbr
 ```
 
 #### Running the `sysinternals` definition file using the `dfe` product:
 
-```
+```bash
 surveyor.py --deffile sysinternals dfe --creds dfe_creds.ini
+```
+
+#### Running a Sigma rule file using the `cbc` product:
+
+```bash
+surveyor.py --sigmarule /path/to/sigma/rule.yml cbc
 ```

common.py

@@ -1,11 +1,16 @@
 import logging
+import os
 from abc import ABC, abstractmethod
-from dataclasses import dataclass, field
-from typing import Tuple, Optional, Any, Union
+from dataclasses import dataclass
+from typing import Tuple, Optional, Any
 
 from help import log_echo
 
 
+class AuthenticationError(Exception):
+    pass
+
+
 @dataclass(eq=True, frozen=True)
 class Tag:
     tag: str
@@ -27,7 +32,7 @@ class Product(ABC):
 
     Subclasses must implement all abstract methods and invoke this class's constructor.
     """
-    product: str = None  # a string describing the product (e.g. cbr/cbth/defender/s1)
+    product: Optional[str] = None  # a string describing the product (e.g. cbr/cbth/defender/s1)
     profile: str  # the profile is used to authenticate to the target platform
     _results: dict[Tag, list[Result]]
     log: logging.Logger
@@ -120,7 +125,7 @@ def _add_results(self, results: list[Result], tag: Optional[Tag] = None):
         Add results to the result store.
         """
         if not tag:
-            tag = '_default'
+            tag = Tag('_default')
 
         if tag not in self._results:
             self._results[tag] = list()
@@ -132,3 +137,88 @@ def _echo(self, message: str, level: int = logging.DEBUG):
         Write a message to STDOUT and the debug log stream.
         """
         log_echo(message, self.log, level, use_tqdm=self._tqdm_echo)
+
+def sigma_translation(product: str, sigma_rules: list, pq: bool = False) -> dict:
+    """
+    Translates a list of sigma rules into the target product language
+
+    Parameters
+    ----------
+    product : str
+        Name of target product
+    sigma_rules : list
+        List of files containing sigma rules or YML-formatted strings
+        Does not support a mixed list of files and strings
+    pq : bool
+        Only used for SentinelOne translations (default is False)
+        If true, translates into PowerQuery syntax
+        Otherwise, uses DeepVisibility
+    """
+
+    supports_json_ouput = True
+
+    try:
+        from sigma.collection import SigmaCollection # type: ignore
+        from sigma.plugins import SigmaPluginDirectory # type: ignore
+        plugins = SigmaPluginDirectory.default_plugin_directory()
+    except Exception as e:
+        raise e
+
+    if product in ('cbr','cbc'):
+        plugins.get_plugin_by_id('carbonblack').install()
+        from sigma.backends.carbonblack import CarbonBlackBackend # type: ignore
+
+        if product == 'cbr':
+            from sigma.pipelines.carbonblack import CarbonBlackResponse_pipeline as cb_pipeline # type: ignore
+        else:
+            from sigma.pipelines.carbonblack import CarbonBlack_pipeline as cb_pipeline # type: ignore
+
+        backend = CarbonBlackBackend(cb_pipeline())
+    elif product == 's1':
+        if pq:
+            plugins.get_plugin_by_id('sentinelone-pq').install()
+            from sigma.backends.sentinelone_pq import SentinelOnePQBackend # type: ignore
+            backend = SentinelOnePQBackend()
+        else:
+            plugins.get_plugin_by_id('sentinelone').install()
+            from sigma.backends.sentinelone import SentinelOneBackend # type: ignore
+            backend = SentinelOneBackend()
+    elif product == 'dfe':
+        supports_json_ouput = False
+        plugins.get_plugin_by_id('microsoft365defender').install()
+        from sigma.backends.kusto import KustoBackend # type: ignore
+        from sigma.pipelines.microsoft365defender import microsoft_365_defender_pipeline # type: ignore 
+        backend = KustoBackend(microsoft_365_defender_pipeline())
+    elif product == 'cortex':
+        plugins.get_plugin_by_id('cortexxdr').install()
+        from sigma.backends.cortexxdr import CortexXDRBackend # type: ignore
+        backend = CortexXDRBackend()
+
+    are_files = [os.path.isfile(i) for i in sigma_rules]
+
+    if all(are_files): # if all items in the list are files
+        if product == "dfe":
+            rule_collection = [SigmaCollection.load_ruleset([i]) for i in sigma_rules] # load each file as a separate rule collection for DFE
+        else:
+            rule_collection = SigmaCollection.load_ruleset(sigma_rules) # type: ignore
+    elif not any(are_files): # if none of the items in the list are files, assume YML formatted strings
+        if product == 'dfe':
+            rule_collection = [SigmaCollection.from_yaml(i) for i in sigma_rules] # load each YML string as a separate rule collection for DFE
+        else:
+            rule_collection = SigmaCollection.merge([SigmaCollection.from_yaml(i) for i in sigma_rules]) # type: ignore
+    else:
+        logging.error("There appears to be a mix of files and YML strings. Cannot process a mixed list of values. Aborting.")
+        return {'queries': []}
+
+    if supports_json_ouput:
+        return backend.convert(rule_collection, "json") # type: ignore
+    else:
+        results: dict = {"queries":[]}
+        for r in rule_collection:
+            results['queries'].append({
+                'query': backend.convert_rule(r)[0] if product != 'dfe' else backend.convert(r)[0], # type: ignore
+                'id': r.id if product != 'dfe' else r.rules[0].id, # type: ignore
+                'title': r.title if product != 'dfe' else r.rules[0].title, # type: ignore
+                'description': r.description if product != 'dfe' else r.rules[0].description # type: ignore
+            })
+        return results

definitions/Incident_202103_Exchange_Activity.json

@@ -76,11 +76,11 @@
 			"Add-PSSnapin\\ Microsoft.Exchange.Powershell.Snapin"
 		], 
 
-		"process_name":[
-			"UMWorkerProcess.exe AND -(childproc_name:wermgr.exe OR childproc_name:WerFault.exe)", 
-			"UMWorkerProcess.exe AND -(filemod:.txt OR filemod:.cfg OR filemod:cleanup.bin OR filemod:.log)", 
-			"UMWorkerProcess.exe AND (filemod:.php OR filemod:.jsp OR filemod:.js OR filemod:.aspx OR filemod:.asp OR filemod:.asmx OR filemod:.asax OR filemod:.cfm OR filemod:.shtml)", 
-			"UMService.exe AND (filemod:.php OR filemod:.jsp OR filemod:.js OR filemod:.aspx OR filemod:.asp OR filemod:.asmx OR filemod:.asax OR filemod:.cfm OR filemod:.shtml)"
+		"query":[
+			"process_name:UMWorkerProcess.exe AND -(childproc_name:wermgr.exe OR childproc_name:WerFault.exe)", 
+			"process_name:UMWorkerProcess.exe AND -(filemod:.txt OR filemod:.cfg OR filemod:cleanup.bin OR filemod:.log)", 
+			"process_name:UMWorkerProcess.exe AND (filemod:.php OR filemod:.jsp OR filemod:.js OR filemod:.aspx OR filemod:.asp OR filemod:.asmx OR filemod:.asax OR filemod:.cfm OR filemod:.shtml)", 
+			"process_name:UMService.exe AND (filemod:.php OR filemod:.jsp OR filemod:.js OR filemod:.aspx OR filemod:.asp OR filemod:.asmx OR filemod:.asax OR filemod:.cfm OR filemod:.shtml)"
 		]
 
 	}