Merge pull request #67 from carbonblack/develop

Release v1.1.0

Author: avanbrunt-cb

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019 Carbon Black
+Copyright (c) 2020 Carbon Black
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -2,9 +2,9 @@
 [![Coverage Status](https://coveralls.io/repos/github/carbonblack/cbc-binary-toolkit/badge.svg?branch=develop)](https://coveralls.io/github/carbonblack/cbc-binary-toolkit?branch=develop)
 # Carbon Black Cloud Binary Toolkit
 
-**Latest Version:** 1.0.0
+**Latest Version:** 1.1.0
 <br>
-**Release Date:** 06/30/2020
+**Release Date:** 11/20/2020
 
 The Carbon Black Cloud Binary Toolkit provides a system of processing incoming SHA256 hashes by integrating with the Unified Binary Store (UBS) on the Carbon Black Cloud (CBC).
 
@@ -26,7 +26,7 @@ Use of the Carbon Black API is governed by the license found in [LICENSE](LICENS
 
 ## Requirements
 
-The Carbon Black Cloud Binary Toolkit is design to work on Python 3.6 and above.
+The Carbon Black Cloud Binary Toolkit is designed to work on Python 3.6 and above.
 
 All requirements are installed as part of `pip install cbc-binary-toolkit` or if you're planning on pushing changes to the Carbon Black Cloud Binary Toolkit, the following can be used after cloning the repo `pip install -r requirements.txt`
 
@@ -48,7 +48,7 @@ OpenSUSE/SUSE | `zypper install python3-devel`
 
 ### Python Packages
 * argparse
-* cbapi
+* carbon-black-cloud-sdk
 * python-dateutil
 * pyyaml
 * requests

VERSION

@@ -1 +1 @@
-1.0.0
+1.1.0

env.encrypted

@@ -1,2 +1,2 @@
 codeship:v2
-kRPdF6MGTXFNXnwojAVVvbtNZpYRFC+3Q6IzrLEBsJQfipQQSOlLcvisv61Ai118QXo8Z5A4AggZLUlWsXS2wWNtpmQ5+volm+RBlt6ov7LLQTLLfe+CXP5av6CG1uEexozw9QnbDw==
+G3bfaQluHoG25wGmaLA5W7B8j0eIf7ALa9LPIAspND4T4kpoFCs7QUUjXyzAkelIhd9ENpM/4HmhcLOu3x752pyIGwzWgnqBnm0fjXo3yfyQwq3a9mrf1rt9RGRO74MlDlXKj76srg==

requirements.txt

@@ -1,5 +1,5 @@
 # Package dependencies
-cbapi
+carbon-black-cloud-sdk
 python-dateutil
 pyyaml
 requests

setup.py

@@ -11,7 +11,7 @@ def read(fname):
 
 
 install_reqs = [
-    "cbapi",
+    "carbon-black-cloud-sdk",
     "python-dateutil",
     "pyyaml",
     "requests",

src/cbc_binary_toolkit/engine_results.py

@@ -24,8 +24,8 @@
 
 from schema import SchemaError
 from .schemas import EngineResponseSchema, IOCv2SEVSchema
-from cbapi.psc.threathunter import Report
-from cbapi.errors import ObjectNotFoundError
+from cbc_sdk.enterprise_edr import Report
+from cbc_sdk.errors import ObjectNotFoundError
 
 log = logging.getLogger(__name__)
 
@@ -37,7 +37,7 @@ class EngineResults:
     Require Properties:
         engine_name (str): The name of the engine analysis is coming from
         state_manager (cbc_binary_toolkit.state.manager): State management component
-        cbth (CbThreatHunterAPI): CBAPI ThreatHunter API to push reports to Carbon Black Cloud
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API to push reports to Carbon Black Cloud
 
     Description:
         Validates an EngineResponse
@@ -52,7 +52,7 @@ class EngineResults:
     Args:
         engine_name (str): The name of the engine analysis is coming from.
         state_manager (cbc_binary_toolkit.state.builtin.SQLiteBasedPersistor): State management object.
-        cbth (cbapi.CbThreatHunterAPI): CBAPI ThreatHunter API to push reports to Carbon Black Cloud.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API to push reports to Carbon Black Cloud.
 
     Attributes:
         SEVERITY_RANGE (int): Highest severity value expected from Analysis Engine for an IOC.
@@ -62,11 +62,11 @@ class EngineResults:
 
     SEVERITY_RANGE = 10
 
-    def __init__(self, engine_name, state_manager, cbth):
+    def __init__(self, engine_name, state_manager, cbc_api):
         """Engine Results Handler Constructor"""
         self.engine_name = engine_name
         self.state_manager = state_manager
-        self.cbth = cbth
+        self.cbc_api = cbc_api
         # Create range of report levels
         self.iocs = list(list() for i in range(self.SEVERITY_RANGE))
 
@@ -219,7 +219,7 @@ def _send_reports(self, feed_id):
                         "iocs_v2": self.iocs[sev]
                     }
 
-                    report = Report(self.cbth, initial_data=report_meta, feed_id=feed_id)
+                    report = Report(self.cbc_api, initial_data=report_meta, feed_id=feed_id)
                     report.update()
                     log.info(f"Report ({report_meta['title']}) sent to feed {feed_id}")
                     # Clear report items from the database

src/cbc_binary_toolkit/ingestion_component.py

@@ -32,7 +32,7 @@ class IngestionComponent:
 
     Args:
         config (Config): Config object.
-        cb_threat_hunter (cbapi.CbThreatHunterAPI): Carbon Black ThreatHunter API object.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API object.
         state_manager (cbc_binary_toolkit.state.builtin.SQLiteBasedPersistor): State management object.
 
     Attributes:
@@ -42,10 +42,10 @@ class IngestionComponent:
     """
     DEFAULT_EXPIRATION = 3600
 
-    def __init__(self, config, cb_threat_hunter, state_manager):
+    def __init__(self, config, cbc_api, state_manager):
         """Constructor"""
         self.config = config
-        self.cb_threat_hunter = cb_threat_hunter
+        self.cbc_api = cbc_api
         self.state_manager = state_manager
 
     def reload(self):
@@ -107,7 +107,7 @@ def fetch_metadata(self, hashes):
         hashes_copy = copy.deepcopy(hashes)
         # Fetch download url from UBS
         while hashes_copy != []:
-            found.extend(download_hashes(self.cb_threat_hunter,
+            found.extend(download_hashes(self.cbc_api,
                                          hashes_copy[:100],
                                          self.config.get("carbonblackcloud.expiration_seconds",
                                                          self.DEFAULT_EXPIRATION)))
@@ -118,7 +118,7 @@ def fetch_metadata(self, hashes):
 
         # Fetch metadata from UBS
         for download_data in found:
-            metadata = get_metadata(self.cb_threat_hunter, download_data)
+            metadata = get_metadata(self.cbc_api, download_data)
 
             # Save hash entry to state manager
             if metadata:

src/cbc_binary_toolkit/ubs.py

@@ -13,7 +13,7 @@
 
 """Functions to retrieve binaries from Unified Binary Store"""
 
-from cbapi.psc.threathunter.models import Binary, Downloads
+from cbc_sdk.enterprise_edr import Binary, Downloads
 import logging
 import copy
 
@@ -25,7 +25,7 @@ class RedownloadHashes:
     Values and function to redownload any hashes that experienced an error during the initial download attempt.
 
     Args:
-        cbth (cbapi.CbThreatHunterAPI): Carbon Black ThreatHunter object.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API object.
         shas (List[str]): hashes to be redownloaded.
         expiration_seconds (int): Desired timeout for AWS links to binaries.
 
@@ -40,9 +40,9 @@ class RedownloadHashes:
 
     RETRY_LIMIT = 5
 
-    def __init__(self, cbth, shas, expiration_seconds):
+    def __init__(self, cbc_api, shas, expiration_seconds):
         """Redownload Hashes constructor"""
-        self.cb = cbth
+        self.cbc_api = cbc_api
         self.shas = shas
         self.expiration_seconds = expiration_seconds
         self.found = []
@@ -55,8 +55,8 @@ def redownload(self):
             "sha256": self.shas,
             "expiration_seconds": self.expiration_seconds,
         }
-        url = self.urlobject.format(self.cb.credentials.org_key)
-        download = self.cb.post_object(url, body).json()
+        url = self.urlobject.format(self.cbc_api.credentials.org_key)
+        download = self.cbc_api.post_object(url, body).json()
         self.attempt_num += 1
         # save any hashes found on the first retry
         if download["found"]:
@@ -67,7 +67,7 @@ def redownload(self):
 
         while download["error"] and self.attempt_num < self.RETRY_LIMIT:
             body["sha256"] = copy.deepcopy(download["error"])
-            download = self.cb.post_object(url, body).json()
+            download = self.cbc_api.post_object(url, body).json()
 
             if download["found"]:
                 self.found.extend(copy.deepcopy(download["found"]))
@@ -85,12 +85,12 @@ def redownload(self):
                         f"the Unified Binary Store: {self.not_found}")
 
 
-def _download_hashes(cbth, hashes, expiration_seconds):
+def _download_hashes(cbc_api, hashes, expiration_seconds):
     """
     Download hashes from Unified Binary Store.
 
     Args:
-        cbth (cbapi.CbThreatHunterAPI): Carbon Black ThreatHunter object.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API object.
         hashes (List[str]): hashes to be downloaded from Unified Binary Store.
         expiration_seconds (int): Desired timeout for AWS links to binaries.
 
@@ -101,19 +101,19 @@ def _download_hashes(cbth, hashes, expiration_seconds):
     """
     try:
         log.debug("Downloading hashes from Unified Binary Store")
-        downloads = Downloads(cbth, hashes, expiration_seconds)
+        downloads = Downloads(cbc_api, hashes, expiration_seconds)
         return downloads
     except Exception as err:
         log.error(f"Error downloading hashes from Unified Binary Store: {err}")
         return None
 
 
-def _download_binary_metadata(cbth, found_binary):
+def _download_binary_metadata(cbc_api, found_binary):
     """
     Retrieve metadata for a binary found in the Unified Binary Store.
 
     Args:
-        cbth (cbapi.CbThreatHunterAPI): Carbon BlackThreatHunter object.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API object.
         found_binary (Dict): Dictionary with "sha256" and "url" values.
 
     Returns:
@@ -125,7 +125,7 @@ def _download_binary_metadata(cbth, found_binary):
         try:
             log.debug("Downloading metadata information")
             binary_metadata = {"url": found_binary["url"]}
-            th_binary = cbth.select(Binary, found_binary["sha256"])
+            th_binary = cbc_api.select(Binary, found_binary["sha256"])
             if isinstance(th_binary, Binary):
                 binary_metadata.update(th_binary._info)
             return binary_metadata
@@ -137,13 +137,13 @@ def _download_binary_metadata(cbth, found_binary):
         return {}
 
 
-def _validate_download(cbth, download, expiration_seconds):
+def _validate_download(cbc_api, download, expiration_seconds):
     """
     Verifies the presence of Downloads.FoundItem. Retries downloading if there are errors during download.
 
     Args:
-        cbth (CbThreatHunterAPI): Carbon BlackThreatHunter object.
-        download (ThreatHunter.Downloads): May contain found, not_found, and error attributes.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API object.
+        download (cbc_sdk.enterprise_edr.Downloads): May contain found, not_found, and error attributes.
         expiration_seconds (int): Desired timeout for AWS links to binaries.
 
     Returns:
@@ -167,7 +167,7 @@ def _validate_download(cbth, download, expiration_seconds):
         log.warning(f"{len(download.error)} hashes experienced an error while"
                     f" downloading: {download.error}. Retrying download.")
 
-        redownload = RedownloadHashes(cbth, [download.error], expiration_seconds)
+        redownload = RedownloadHashes(cbc_api, [download.error], expiration_seconds)
 
         redownload.redownload()
 
@@ -176,12 +176,12 @@ def _validate_download(cbth, download, expiration_seconds):
     return download_found, redownload
 
 
-def download_hashes(cbth, hashes, expiration_seconds=3600):
+def download_hashes(cbc_api, hashes, expiration_seconds=3600):
     """
     Initiates download of hashes
 
     Args:
-        cbth (cbapi.CbThreatHunterAPI): Carbon BlackThreatHunter object.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API object.
         hashes (List[str]): hashes to be downloaded from Unified Binary Store.
         expiration_seconds (int, optional): Desired timeout for AWS links to binaries.
 
@@ -190,15 +190,15 @@ def download_hashes(cbth, hashes, expiration_seconds=3600):
         Empty list if an error occurred during download.
 
     Examples:
-        >>> download_hashes(cbth, ["0995f71c34f613207bc39ed4fcc1bbbee396a543fa1739656f7ddf70419309fc"])
+        >>> download_hashes(cbc_api, ["0995f71c34f613207bc39ed4fcc1bbbee396a543fa1739656f7ddf70419309fc"])
 
     """
     if not hashes:
         log.error("No hashes supplied to download_hashes.")
         return list()
-    download = _download_hashes(cbth, hashes, expiration_seconds)
+    download = _download_hashes(cbc_api, hashes, expiration_seconds)
 
-    checked_download, retried_download = _validate_download(cbth, download, expiration_seconds)
+    checked_download, retried_download = _validate_download(cbc_api, download, expiration_seconds)
 
     if not checked_download:
         log.error("Unable to retrieve binaries from the Unified Binary Store.")
@@ -213,12 +213,12 @@ def download_hashes(cbth, hashes, expiration_seconds=3600):
     return found_hashes
 
 
-def get_metadata(cbth, binary):
+def get_metadata(cbc_api, binary):
     """
     Initiates download of binary metadata from Unified Binary Store.
 
     Args:
-        cbth (cbapi.CbThreatHunterAPI): Carbon Black ThreatHunter object.
+        cbc_api (cbc_sdk.CBCloudAPI): Carbon Black Cloud API object.
         binary (Dict): Dictionary with "sha256" and "url" values.
 
     Returns:
@@ -230,7 +230,7 @@ def get_metadata(cbth, binary):
         return {}
     else:
         try:
-            return _download_binary_metadata(cbth, binary)
+            return _download_binary_metadata(cbc_api, binary)
         except Exception as err:
             log.error(f"Failed to download metadata: {err}")
             return {}

src/cbc_binary_toolkit_examples/tools/analysis_util.py

@@ -33,7 +33,7 @@
 from cbc_binary_toolkit.engine import LocalEngineManager
 from cbc_binary_toolkit.state import StateManager
 
-from cbapi import CbThreatHunterAPI
+from cbc_sdk import CBCloudAPI
 
 DEFAULT_LOG_LEVEL = "INFO"
 
@@ -54,7 +54,7 @@ def __init__(self, default_install):
         """Constructor for the analysis utility class"""
         self.default_install = default_install
         self.config = None
-        self.cbapi = None
+        self.cbc_api = None
 
         # Create argument parser
         self._parser = argparse.ArgumentParser()
@@ -98,17 +98,17 @@ def _init_components(self):
             log.debug(traceback.format_exc())
             state_manager = None
 
-        cbth = self.cbapi
-        if cbth is None:
-            cbth = CbThreatHunterAPI(url=self.config.get("carbonblackcloud.url"),
-                                     org_key=self.config.get("carbonblackcloud.org_key"),
-                                     token=self.config.get("carbonblackcloud.api_token"),
-                                     ssl_verify=self.config.get("carbonblackcloud.ssl_verify"))
+        cbc_api = self.cbc_api
+        if cbc_api is None:
+            cbc_api = CBCloudAPI(url=self.config.get("carbonblackcloud.url"),
+                              org_key=self.config.get("carbonblackcloud.org_key"),
+                              token=self.config.get("carbonblackcloud.api_token"),
+                              ssl_verify=self.config.get("carbonblackcloud.ssl_verify"))
 
         deduplicate = DeduplicationComponent(self.config, state_manager)
-        ingest = IngestionComponent(self.config, cbth, state_manager)
+        ingest = IngestionComponent(self.config, cbc_api, state_manager)
 
-        results_engine = EngineResults(self.config.get("engine.name"), state_manager, cbth)
+        results_engine = EngineResults(self.config.get("engine.name"), state_manager, cbc_api)
         if self.config.get("engine.type") == "local":
             try:
                 engine_manager = LocalEngineManager(self.config)