From 937f8ad9a68ade7a1878f4d5b93ae0d9a9c50607 Mon Sep 17 00:00:00 2001 From: Jay <51702743+jayjb@users.noreply.github.com> Date: Mon, 14 Aug 2023 14:38:12 +0200 Subject: [PATCH] Add pre-commit config (#293) * Add pre-commit config * . * . * . * . * . * Fixes fo flake8 --- .flake8 | 5 + .github/workflows/opencanary_tests.yml | 13 + .github/workflows/publish.yml | 6 +- .pre-commit-config.yaml | 29 + .readthedocs.yaml | 2 +- MANIFEST.in | 2 +- README.md | 2 +- bin/opencanary-correlator | 3 +- bin/opencanary.tac | 103 +- .../generate_macOS_launchctl_service_files.py | 119 +- docs/alerts/email.rst | 2 +- docs/conf.py | 163 +-- docs/services/mssql.rst | 1 - docs/services/mysql.rst | 1 - docs/services/webserver.rst | 3 +- docs/services/windows.rst | 4 +- docs/starting/correlator.rst | 2 +- docs/starting/opencanary.rst | 3 +- opencanary.service | 2 +- opencanary/__init__.py | 2 +- opencanary/config.py | 97 +- opencanary/honeycred.py | 11 +- opencanary/iphelper.py | 8 +- opencanary/logger.py | 256 ++-- opencanary/modules/__init__.py | 80 +- .../skin/nasLogin/static/css/xtheme-gray.css | 12 +- .../data/http/skin/nasLogin/static/js/misc.js | 1 - .../data/httpproxy/skin/squid/auth.html | 5 +- opencanary/modules/des.py | 1099 ++++++++++++++--- opencanary/modules/example0.py | 14 +- opencanary/modules/example1.py | 11 +- opencanary/modules/ftp.py | 29 +- opencanary/modules/git.py | 24 +- opencanary/modules/http.py | 89 +- opencanary/modules/httpproxy.py | 75 +- opencanary/modules/https.py | 6 +- opencanary/modules/mssql.py | 250 ++-- opencanary/modules/mysql.py | 85 +- opencanary/modules/ntp.py | 36 +- opencanary/modules/portscan.py | 187 +-- opencanary/modules/redis.py | 529 ++++---- opencanary/modules/samba.py | 91 +- opencanary/modules/sip.py | 25 +- opencanary/modules/snmp.py | 19 +- opencanary/modules/ssh.py | 236 ++-- opencanary/modules/tcpbanner.py | 203 ++- opencanary/modules/telnet.py | 50 +- opencanary/modules/testpdf.py | 102 -- opencanary/modules/tftp.py | 32 +- opencanary/modules/vnc.py | 117 +- opencanary/test/logger.py | 2 +- opencanary/test/module_test.py | 247 ++-- run.sh | 1 - setup.py | 64 +- 54 files changed, 2847 insertions(+), 1713 deletions(-) create mode 100644 .flake8 create mode 100644 .pre-commit-config.yaml delete mode 100644 opencanary/modules/testpdf.py diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..e1160ade --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = E501, W503, E203 +exclude = .git,__pycache__,docs/conf.py,build,dist,opencanary/modules/des.py +max-complexity = 10 +min_python_version = 3.9 diff --git a/.github/workflows/opencanary_tests.yml b/.github/workflows/opencanary_tests.yml index 3d9e212c..4ce91fa1 100644 --- a/.github/workflows/opencanary_tests.yml +++ b/.github/workflows/opencanary_tests.yml @@ -5,6 +5,19 @@ on: - "pull_request" jobs: + precommit_tests: + runs-on: "ubuntu-20.04" + steps: + - name: "Check out repository code" + uses: "actions/checkout@v3" + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install pre-commit + run: pip install pre-commit + - name: Check pre-commit is happy + run: pre-commit run --all-files opencanary_tests: strategy: matrix: diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f7af6ada..ce97d0ed 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,7 +15,7 @@ jobs: id-token: write # IMPORTANT: this permission is mandatory for trusted publishing steps: # retrieve your distributions here - - name: Set up Python + - name: Set up Python uses: actions/setup-python@v3 - name: "Check out repository code" uses: "actions/checkout@v3" @@ -37,9 +37,9 @@ jobs: else echo "Versions do not match - not publishing" echo "Opencanary version is: $version_to_release" - echo "Git tag is: $tag_name -> $tag_name_without_v" + echo "Git tag is: $tag_name -> $tag_name_without_v" exit 1 fi - + - name: Publish package distributions to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c38b2790 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +exclude: > + (?x)^( + dist/| + .devcontainer/devcontainer.json + ) +fail_fast: true +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.1.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-docstring-first + - id: check-json + - id: check-added-large-files + - id: check-yaml + - id: debug-statements + # - id: no-commit-to-branch + # # GitHub only allows branch protection for teams or enterprise. + # args: ['--pattern', '^(?!T\d+.*)'] +- repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black +- repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 + hooks: + - id: flake8 + additional_dependencies: [flake8-typing-imports==1.12.0] diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 239801e6..b7f16e6b 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -22,4 +22,4 @@ sphinx: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: docs/requirements.txt \ No newline at end of file + - requirements: docs/requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 0caa3fed..d4012bfc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,4 +8,4 @@ recursive-exclude docs * exclude Dockerfile.latest exclude Dockerfile.* exclude docker-compose.yml -exclude .gitignore \ No newline at end of file +exclude .gitignore diff --git a/README.md b/README.md index b257687e..6d963cdf 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ NOTE: The portscan module is automatically disabled for Dockerised OpenCanary. > Requires [Docker](https://docs.docker.com/get-docker/) installed. -NOTE: The portscan module is automatically disabled for Dockerised OpenCanary. +NOTE: The portscan module is automatically disabled for Dockerised OpenCanary. 1. Edit the `data/.opencanary.conf` file to enable, disable or customize the services that will run. diff --git a/bin/opencanary-correlator b/bin/opencanary-correlator index 07f1d25d..85bc7f06 100755 --- a/bin/opencanary-correlator +++ b/bin/opencanary-correlator @@ -3,5 +3,4 @@ from opencanary.correlator import main if __name__ == "__main__": - main() - + main() diff --git a/bin/opencanary.tac b/bin/opencanary.tac index b3c1f0df..c495e4b2 100644 --- a/bin/opencanary.tac +++ b/bin/opencanary.tac @@ -1,13 +1,7 @@ import traceback -# import warnings -# warnings.filterwarnings("ignore", category=DeprecationWarning) -def warn(*args, **kwargs): - pass import warnings -warnings.warn = warn +import sys from twisted.application import service -from twisted.application import internet -from twisted.internet.protocol import Factory from pkg_resources import iter_entry_points from opencanary.config import config, is_docker @@ -29,8 +23,16 @@ from opencanary.modules.redis import CanaryRedis from opencanary.modules.tcpbanner import CanaryTCPBanner from opencanary.modules.rdp import CanaryRDP -#from opencanary.modules.example0 import CanaryExample0 -#from opencanary.modules.example1 import CanaryExample1 + +def warn(*args, **kwargs): + pass + + +warnings.warn = warn + + +# from opencanary.modules.example0 import CanaryExample0 +# from opencanary.modules.example1 import CanaryExample1 ENTRYPOINT = "canary.usermodule" MODULES = [ @@ -54,91 +56,94 @@ MODULES = [ # CanaryExample1, ] -if config.moduleEnabled('snmp'): +if config.moduleEnabled("snmp"): try: - #Module need Scapy, but the rest of OpenCanary doesn't + # Module need Scapy, but the rest of OpenCanary doesn't from opencanary.modules.snmp import CanarySNMP + MODULES.append(CanarySNMP) except ImportError: print("Can't import SNMP. Please ensure you have Scapy installed.") pass # NB: imports below depend on inotify, only available on linux -import sys if sys.platform.startswith("linux"): from opencanary.modules.samba import CanarySamba + MODULES.append(CanarySamba) - if config.moduleEnabled('portscan') and is_docker(): + if config.moduleEnabled("portscan") and is_docker(): # Remove portscan if running in DOCKER (specified in Dockerfile) print("Can't use portscan in Docker. Portscan module disabled.") else: from opencanary.modules.portscan import CanaryPortscan + MODULES.append(CanaryPortscan) logger = getLogger(config) -def start_mod(application, klass): + +def start_mod(application, klass): # noqa: C901 try: obj = klass(config=config, logger=logger) - except Exception as e: - err = 'Failed to instantiate instance of class %s in %s. %s' % ( + except Exception: + err = "Failed to instantiate instance of class %s in %s. %s" % ( klass.__name__, klass.__module__, - traceback.format_exc() + traceback.format_exc(), ) - logMsg({'logdata': err}) + logMsg({"logdata": err}) return - if hasattr(obj, 'startYourEngines'): + if hasattr(obj, "startYourEngines"): try: obj.startYourEngines() - msg = 'Ran startYourEngines on class %s in %s' % ( + msg = "Ran startYourEngines on class %s in %s" % ( klass.__name__, - klass.__module__ - ) - logMsg({'logdata': msg}) + klass.__module__, + ) + logMsg({"logdata": msg}) - except Exception as e: - err = 'Failed to run startYourEngines on %s in %s. %s' % ( + except Exception: + err = "Failed to run startYourEngines on %s in %s. %s" % ( klass.__name__, klass.__module__, - traceback.format_exc() + traceback.format_exc(), ) - logMsg({'logdata': err}) - elif hasattr(obj, 'getService'): + logMsg({"logdata": err}) + elif hasattr(obj, "getService"): try: service = obj.getService() if not isinstance(service, list): service = [service] for s in service: s.setServiceParent(application) - msg = 'Added service from class %s in %s to fake' % ( + msg = "Added service from class %s in %s to fake" % ( klass.__name__, - klass.__module__ - ) - logMsg({'logdata': msg}) - except Exception as e: - err = 'Failed to add service from class %s in %s. %s' % ( + klass.__module__, + ) + logMsg({"logdata": msg}) + except Exception: + err = "Failed to add service from class %s in %s. %s" % ( klass.__name__, klass.__module__, - traceback.format_exc() + traceback.format_exc(), ) - logMsg({'logdata': err}) + logMsg({"logdata": err}) else: - err = 'The class %s in %s does not have any required starting method.' % ( + err = "The class %s in %s does not have any required starting method." % ( klass.__name__, - klass.__module__ + klass.__module__, ) - logMsg({'logdata': err}) + logMsg({"logdata": err}) + def logMsg(msg): data = {} -# data['src_host'] = device_name -# data['dst_host'] = node_id - data['logdata'] = {'msg': msg} + data["logdata"] = {"msg": msg} logger.log(data, retry=False) + application = service.Application("opencanaryd") # List of modules to start @@ -150,12 +155,12 @@ for ep in iter_entry_points(ENTRYPOINT): try: klass = ep.load(require=False) start_modules.append(klass) - except Exception as e: - err = 'Failed to load class from the entrypoint: %s. %s' % ( + except Exception: + err = "Failed to load class from the entrypoint: %s. %s" % ( str(ep), - traceback.format_exc() - ) - logMsg({'logdata': err}) + traceback.format_exc(), + ) + logMsg({"logdata": err}) # Add only enabled modules start_modules.extend(filter(lambda m: config.moduleEnabled(m.NAME), MODULES)) @@ -163,5 +168,5 @@ start_modules.extend(filter(lambda m: config.moduleEnabled(m.NAME), MODULES)) for klass in start_modules: start_mod(application, klass) -msg = 'Canary running!!!' -logMsg({'logdata': msg}) +msg = "Canary running!!!" +logMsg({"logdata": msg}) diff --git a/build_scripts/generate_macOS_launchctl_service_files.py b/build_scripts/generate_macOS_launchctl_service_files.py index 9addba35..458ee569 100755 --- a/build_scripts/generate_macOS_launchctl_service_files.py +++ b/build_scripts/generate_macOS_launchctl_service_files.py @@ -13,67 +13,75 @@ from functools import partial from os import chmod, pardir, path from os.path import dirname, join, realpath -from shutil import copyfile from subprocess import CalledProcessError, check_output from pkg_resources import resource_filename -OPENCANARY = 'opencanary' -LAUNCH_DAEMONS_DIR = '/Library/LaunchDaemons' -DEFAULT_SERVICE_NAME = 'com.thinkst.opencanary' -CONFIG_FILE_BASENAME = 'opencanary.conf' -DEFAULT_CONFIG_FILE = resource_filename('opencanary', 'data/settings.json') +OPENCANARY = "opencanary" +LAUNCH_DAEMONS_DIR = "/Library/LaunchDaemons" +DEFAULT_SERVICE_NAME = "com.thinkst.opencanary" +CONFIG_FILE_BASENAME = "opencanary.conf" +DEFAULT_CONFIG_FILE = resource_filename("opencanary", "data/settings.json") # opencanary dirs OPENCANARY_DIR = realpath(join(dirname(__file__), pardir)) -OPENCANARY_BIN_DIR = join(OPENCANARY_DIR, 'bin') -VENV_DIR = join(OPENCANARY_DIR, 'env') -VENV_BIN_DIR = join(VENV_DIR, 'bin') -DEFAULT_LOG_DIR = join(OPENCANARY_DIR, 'log') +OPENCANARY_BIN_DIR = join(OPENCANARY_DIR, "bin") +VENV_DIR = join(OPENCANARY_DIR, "env") +VENV_BIN_DIR = join(VENV_DIR, "bin") +DEFAULT_LOG_DIR = join(OPENCANARY_DIR, "log") # daemon config -DAEMON_CONFIG_DIR = '/etc/opencanaryd' +DAEMON_CONFIG_DIR = "/etc/opencanaryd" DAEMON_CONFIG_PATH = join(DAEMON_CONFIG_DIR, CONFIG_FILE_BASENAME) -DAEMON_PATH = join(VENV_BIN_DIR, 'opencanaryd') +DAEMON_PATH = join(VENV_BIN_DIR, "opencanaryd") DAEMON_RUNTIME_OPTIONS = "--dev" # This script writes to the launchctl/ folder -LAUNCHCTL_DIR = join(OPENCANARY_DIR, 'launchctl') +LAUNCHCTL_DIR = join(OPENCANARY_DIR, "launchctl") # Homebrew (TODO: is this necessary?) try: - homebrew_bin_dir = join(check_output(['brew', '--prefix']).decode().rstrip(), 'bin') + homebrew_bin_dir = join(check_output(["brew", "--prefix"]).decode().rstrip(), "bin") except CalledProcessError as e: print(f"Couldn't get homebrew install location: {e}") sys.exit() # Load opencanary.conf default config -with open(DEFAULT_CONFIG_FILE, 'r') as file: +with open(DEFAULT_CONFIG_FILE, "r") as file: config = json.load(file) - canaries = [k.split(".")[0] for k in config.keys() if re.match("[a-z]+\\.enabled", k)] + canaries = [ + k.split(".")[0] for k in config.keys() if re.match("[a-z]+\\.enabled", k) + ] # Parse arguments. parser = ArgumentParser( - description='Generate .plist, opencanary.conf, and scripts to bootstrap opencanary as a launchctl daemon.', - formatter_class=ArgumentDefaultsHelpFormatter + description="Generate .plist, opencanary.conf, and scripts to bootstrap opencanary as a launchctl daemon.", + formatter_class=ArgumentDefaultsHelpFormatter, ) -parser.add_argument('--service-name', - help='string you would like launchctl to use as the name of the opencanary service', - metavar='NAME', - default=DEFAULT_SERVICE_NAME) +parser.add_argument( + "--service-name", + help="string you would like launchctl to use as the name of the opencanary service", + metavar="NAME", + default=DEFAULT_SERVICE_NAME, +) -parser.add_argument('--log-output-dir', - help='opencanary will write its logs to files in DIR when the service is running', - metavar='DIR', - default=DEFAULT_LOG_DIR) +parser.add_argument( + "--log-output-dir", + help="opencanary will write its logs to files in DIR when the service is running", + metavar="DIR", + default=DEFAULT_LOG_DIR, +) -parser.add_argument('--canary', action='append', - help=f'enable canary service in the generated opencanary.conf file ' + \ - '(can be supplied more than once)', - choices=canaries, - dest='canaries') +parser.add_argument( + "--canary", + action="append", + help="enable canary service in the generated opencanary.conf file " + + "(can be supplied more than once)", + choices=canaries, + dest="canaries", +) args = parser.parse_args() @@ -87,13 +95,15 @@ # File builders build_launchctl_dir_path = partial(join, LAUNCHCTL_DIR) -build_logfile_name = lambda log_name: join(args.log_output_dir, f"opencanary.{log_name}.log") +build_logfile_name = lambda log_name: join( # noqa: E731 + args.log_output_dir, f"opencanary.{log_name}.log" +) # daemon launcher script launcher_script = build_launchctl_dir_path(f"launch_{args.service_name}.sh") -with open(launcher_script, 'w') as file: +with open(launcher_script, "w") as file: file.write(f'. "{VENV_BIN_DIR}/activate"\n') file.write(f'"{DAEMON_PATH}" {DAEMON_RUNTIME_OPTIONS}\n') @@ -102,20 +112,20 @@ plist_output_file = build_launchctl_dir_path(plist_basename) plist_contents = { - 'Label': args.service_name, - 'RunAtLoad': True, - 'KeepAlive': True, - 'WorkingDirectory': VENV_BIN_DIR, - 'StandardOutPath': build_logfile_name('err'), - 'StandardErrorPath': build_logfile_name('out'), - 'EnvironmentVariables': { - 'PATH': f"{VENV_BIN_DIR}:{homebrew_bin_dir}:/usr/bin:/bin", - 'VIRTUAL_ENV': VENV_DIR + "Label": args.service_name, + "RunAtLoad": True, + "KeepAlive": True, + "WorkingDirectory": VENV_BIN_DIR, + "StandardOutPath": build_logfile_name("err"), + "StandardErrorPath": build_logfile_name("out"), + "EnvironmentVariables": { + "PATH": f"{VENV_BIN_DIR}:{homebrew_bin_dir}:/usr/bin:/bin", + "VIRTUAL_ENV": VENV_DIR, }, - 'ProgramArguments': [launcher_script] + "ProgramArguments": [launcher_script], } -with open(plist_output_file, 'wb+') as _plist_file: +with open(plist_output_file, "wb+") as _plist_file: plistlib.dump(plist_contents, _plist_file) @@ -124,7 +134,7 @@ config[f"{canary}.enabled"] = canary in args.canaries log_handlers = config["logger"]["kwargs"]["handlers"] -log_handlers["file"]["filename"] = build_logfile_name('run') +log_handlers["file"]["filename"] = build_logfile_name("run") # TODO: This config doesn't work even though direct calls to syslog do # log_handlers["syslog-unix"] = { @@ -139,32 +149,35 @@ config_output_file = build_launchctl_dir_path(CONFIG_FILE_BASENAME) -with open(config_output_file, 'w') as file: +with open(config_output_file, "w") as file: file.write(json.dumps(config, indent=4)) # service bootstrap script -install_service_script = build_launchctl_dir_path(f"install_service_{args.service_name}.sh") +install_service_script = build_launchctl_dir_path( + f"install_service_{args.service_name}.sh" +) daemon_plist_path = join(LAUNCH_DAEMONS_DIR, plist_basename) -with open(install_service_script, 'w') as file: +with open(install_service_script, "w") as file: script_contents = [ - f"set -e\n\n" - f"chown root '{launcher_script}'", + f"set -e\n\n" f"chown root '{launcher_script}'", f"mkdir -p '{DAEMON_CONFIG_DIR}'", f"cp '{config_output_file}' {DAEMON_CONFIG_PATH}", f"cp '{plist_output_file}' {LAUNCH_DAEMONS_DIR}", f"launchctl bootstrap system '{daemon_plist_path}'", - "" + "", ] file.write("\n".join(script_contents)) # uninstall/bootout script -uninstall_service_script = build_launchctl_dir_path(f"uninstall_service_{args.service_name}.sh") +uninstall_service_script = build_launchctl_dir_path( + f"uninstall_service_{args.service_name}.sh" +) -with open(uninstall_service_script, 'w') as file: +with open(uninstall_service_script, "w") as file: file.write(f"launchctl bootout system/{args.service_name}\n") diff --git a/docs/alerts/email.rst b/docs/alerts/email.rst index e0e5b446..7b5f1ca6 100644 --- a/docs/alerts/email.rst +++ b/docs/alerts/email.rst @@ -11,7 +11,7 @@ In the configurations below, set these configuration variables: * **subject** - The email's subject. * **credentials** - Optional parameter, if the SMTP server requires authentication. * **secure** - Optional parameter if TLS support is mandatory or wanted. - + More information can be found on the `PyLogger page `_. Send to a Gmail address diff --git a/docs/conf.py b/docs/conf.py index 704c2765..2397db1f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -19,48 +19,48 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['myst_parser'] +extensions = ["myst_parser"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # source_suffix = ['.rst', '.md'] source_suffix = { - '.rst': 'restructuredtext', - '.md': 'markdown', + ".rst": "restructuredtext", + ".md": "markdown", } # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'OpenCanary' -copyright = u'Thinkst Canary' -author = u'Thinkst Applied Research' +project = "OpenCanary" +copyright = 'Thinkst Canary' +author = "Thinkst Applied Research" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.9' +version = "0.9" # The full version, including alpha/beta/rc tags. -release = '0.9' +release = "0.9" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -71,37 +71,37 @@ # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False @@ -111,156 +111,155 @@ # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'alabaster' +html_theme = "alabaster" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -html_logo = 'logo.png' +html_logo = "logo.png" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Language to be used for generating the HTML full-text search index. # Sphinx supports the following languages: # 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' # 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' -#html_search_language = 'en' +# html_search_language = 'en' # A dictionary with options for the search language support, empty by default. # Now only 'ja' uses this config value -#html_search_options = {'type': 'default'} +# html_search_options = {'type': 'default'} # The name of a javascript file (relative to the configuration directory) that # implements a search results scorer. If empty, the default will be used. -#html_search_scorer = 'scorer.js' +# html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'OpenCanarydoc' +htmlhelp_basename = "OpenCanarydoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', - -# Latex figure (float) alignment -#'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + #'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + #'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + #'preamble': '', + # Latex figure (float) alignment + #'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'OpenCanary.tex', u'OpenCanary Documentation', - u'Thinkst Applied Research', 'manual'), + ( + master_doc, + "OpenCanary.tex", + "OpenCanary Documentation", + "Thinkst Applied Research", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). -man_pages = [ - (master_doc, 'opencanary', u'OpenCanary Documentation', - [author], 1) -] +man_pages = [(master_doc, "opencanary", "OpenCanary Documentation", [author], 1)] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -269,19 +268,25 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'OpenCanary', u'OpenCanary Documentation', - author, 'OpenCanary', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "OpenCanary", + "OpenCanary Documentation", + author, + "OpenCanary", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False diff --git a/docs/services/mssql.rst b/docs/services/mssql.rst index 111d291e..3942a4d0 100644 --- a/docs/services/mssql.rst +++ b/docs/services/mssql.rst @@ -13,4 +13,3 @@ Inside ~/.opencanary.conf: "rdp.port", 3389, // [..] # logging configuration } - diff --git a/docs/services/mysql.rst b/docs/services/mysql.rst index 69393030..b4a76ff7 100644 --- a/docs/services/mysql.rst +++ b/docs/services/mysql.rst @@ -14,4 +14,3 @@ Inside ~/.opencanary.conf: "ssh.version": "SSH-2.0-OpenSSH_5.1p1 Debian-4", // [..] # logging configuration } - diff --git a/docs/services/webserver.rst b/docs/services/webserver.rst index d3ee60c6..d71d8846 100644 --- a/docs/services/webserver.rst +++ b/docs/services/webserver.rst @@ -5,7 +5,7 @@ Inside ~/.opencanary.conf: .. code-block:: json - { + { "ftp.banner": "FTP server ready", "ftp.enabled": true, "ftp.port":21, @@ -28,4 +28,3 @@ Inside ~/.opencanary.conf: "ssh.version": "SSH-2.0-OpenSSH_5.1p1 Debian-4", // [..] # logging configuration } - diff --git a/docs/services/windows.rst b/docs/services/windows.rst index 8c7454a0..8c912d22 100644 --- a/docs/services/windows.rst +++ b/docs/services/windows.rst @@ -13,7 +13,7 @@ Inside ~/.opencanary.conf: "smb.enabled": true } -Below is an example of an `smb.conf` for a Samba installation, +Below is an example of an `smb.conf` for a Samba installation, .. code-block:: dosini @@ -67,6 +67,6 @@ to use rsyslog, we will edit the `/etc/rsyslog.conf` file. Below are two lines w local7.* /var/log/samba-audit.log This will redirect any message of facility local7 to your `/var/log/samba-audit.log` file, which will be -watched by our OpenCanary daemon. +watched by our OpenCanary daemon. Please note this is all written up in the GitHub README.md diff --git a/docs/starting/correlator.rst b/docs/starting/correlator.rst index d645c01c..493fc63f 100644 --- a/docs/starting/correlator.rst +++ b/docs/starting/correlator.rst @@ -63,7 +63,7 @@ To configure OpenCanary daemons to send their events to the correlator, edit the } } } - + Troubleshooting --------------- diff --git a/docs/starting/opencanary.rst b/docs/starting/opencanary.rst index 6c9aa6ea..33c7df0c 100644 --- a/docs/starting/opencanary.rst +++ b/docs/starting/opencanary.rst @@ -55,7 +55,7 @@ With that in place, we can run the daemon and test that it logs a failed FTP log $ cat /var/tmp/opencanary.log [...] {"dst_host": "127.0.0.1", "dst_port": 21, "local_time": "2015-07-20 13:38:21.281259", "logdata": {"PASSWORD": "default", "USERNAME": "admin"}, "logtype": 2000, "node_id": "opencanary-0", "src_host": "127.0.0.1", "src_port": 49635} - + Troubleshooting --------------- @@ -77,4 +77,3 @@ You may also easily restart the service using, .. code-block:: sh $ opencanaryd --restart - diff --git a/opencanary.service b/opencanary.service index 8b8a9f88..a9197ec0 100644 --- a/opencanary.service +++ b/opencanary.service @@ -12,4 +12,4 @@ ExecStart=/bin/opencanaryd --start ExecStop=/bin/opencanaryd --stop [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target diff --git a/opencanary/__init__.py b/opencanary/__init__.py index fea89013..3e2f46a3 100644 --- a/opencanary/__init__.py +++ b/opencanary/__init__.py @@ -1 +1 @@ -__version__="0.9.0" +__version__ = "0.9.0" diff --git a/opencanary/config.py b/opencanary/config.py index ce551e36..7f4e57da 100644 --- a/opencanary/config.py +++ b/opencanary/config.py @@ -1,11 +1,16 @@ from six import iteritems -import os, sys, json, copy, socket, itertools, string, subprocess +import os +import sys +import json +import itertools +import string +import subprocess from os.path import expanduser from pkg_resources import resource_filename from pathlib import Path -SAMPLE_SETTINGS = resource_filename(__name__, 'data/settings.json') -SETTINGS = 'opencanary.conf' +SAMPLE_SETTINGS = resource_filename(__name__, "data/settings.json") +SETTINGS = "opencanary.conf" def expand_vars(var): @@ -20,17 +25,29 @@ def expand_vars(var): return os.path.expandvars(var) return var + def is_docker(): - cgroup = Path('/proc/self/cgroup') - return Path('/.dockerenv').is_file() or cgroup.is_file() and 'docker' in cgroup.read_text() + cgroup = Path("/proc/self/cgroup") + return ( + Path("/.dockerenv").is_file() + or cgroup.is_file() + and "docker" in cgroup.read_text() + ) + class Config: def __init__(self, configfile=SETTINGS): self.__config = None self.__configfile = configfile - files = [configfile, "%s/.%s" % (expanduser("~"), configfile), "/etc/opencanaryd/%s"%configfile] - print("** We hope you enjoy using OpenCanary. For more open source Canary goodness, head over to canarytokens.org. **") + files = [ + configfile, + "%s/.%s" % (expanduser("~"), configfile), + "/etc/opencanaryd/%s" % configfile, + ] + print( + "** We hope you enjoy using OpenCanary. For more open source Canary goodness, head over to canarytokens.org. **" + ) for fname in files: try: with open(fname, "r") as f: @@ -42,11 +59,15 @@ def __init__(self, configfile=SETTINGS): print("[-] Failed to open %s for reading (%s)" % (fname, e)) except ValueError as e: print("[-] Failed to decode json from %s (%s)" % (fname, e)) - subprocess.call("cp -r %s /var/tmp/config-err-$(date +%%s)" % fname, shell=True) + subprocess.call( + "cp -r %s /var/tmp/config-err-$(date +%%s)" % fname, shell=True + ) except Exception as e: print("[-] An error occurred loading %s (%s)" % (fname, e)) if self.__config is None: - print('No config file found. Please create one with "opencanaryd --copyconfig"') + print( + 'No config file found. Please create one with "opencanaryd --copyconfig"' + ) sys.exit(1) def moduleEnabled(self, module_name): @@ -64,7 +85,7 @@ def getVal(self, key, default=None): return default raise e - def setValues(self, params): + def setValues(self, params): # noqa: C901 """Set all the valid values in params and return a list of errors for invalid""" # silently ensure that node_id and mac are not modified via web @@ -75,14 +96,24 @@ def setValues(self, params): # if dhcp is enabled, ignore the static ip settings if params.get("device.dhcp.enabled", False): - static = ["device.ip_address", "device.netmask", - "device.gw", "device.dns1", "device.dns2"] + static = [ + "device.ip_address", + "device.netmask", + "device.gw", + "device.dns1", + "device.dns2", + ] for k in static: if k in params: del params[k] # for each section, if disabled, delete ignore section's settings - disabled_modules = tuple(filter(lambda m: not params.get("%s.enabled" % m, False), ["ftp", "ssh", "smb", "http"])) + disabled_modules = tuple( + filter( + lambda m: not params.get("%s.enabled" % m, False), + ["ftp", "ssh", "smb", "http"], + ) + ) for k in params.keys(): if not k.endswith("enabled") and k.startswith(disabled_modules): del params[k] @@ -90,9 +121,9 @@ def setValues(self, params): # test options indpenedently for validity errors = [] - for key,value in iteritems(params): + for key, value in iteritems(params): try: - self.valid(key,value) + self.valid(key, value) except ConfigException as e: errors.append(e) @@ -100,7 +131,7 @@ def setValues(self, params): ports = {k: v for k, v in iteritems(self.__config) if k.endswith(".port")} newports = {k: v for k, v in iteritems(params) if k.endswith(".port")} ports.update(newports) - ports = [(port,setting) for setting, port in iteritems(ports)] + ports = [(port, setting) for setting, port in iteritems(ports)] ports.sort() for port, settings in itertools.groupby(ports, lambda x: x[0]): @@ -133,7 +164,7 @@ def setVal(self, key, val): if e.key == key: raise e - def valid(self, key, val): + def valid(self, key, val): # noqa: C901 """ Test an the validity of an individual setting Raise config error message on failure. @@ -142,10 +173,12 @@ def valid(self, key, val): if key.endswith(".enabled"): if not ((val is True) or (val is False)): - raise ConfigException(key, "Boolean setting is not True or False (%s)" % val) + raise ConfigException( + key, "Boolean setting is not True or False (%s)" % val + ) if key.endswith(".port"): - if (not isinstance(val,int)) or val < 1 or val > 65535: + if (not isinstance(val, int)) or val < 1 or val > 65535: raise ConfigException(key, "Invalid port number (%s)" % val) # Max length of SSH version string is 255 chars including trailing CR and LF @@ -165,7 +198,9 @@ def valid(self, key, val): if not f["type"]: raise ConfigException(key, "File type cannot be empty") if f["type"] not in extensions: - raise ConfigException(key, "Extension %s is not supported" % f["type"]) + raise ConfigException( + key, "Extension %s is not supported" % f["type"] + ) if key == "device.name": allowed_chars = string.ascii_letters + string.digits + "+-#_" @@ -175,7 +210,10 @@ def valid(self, key, val): elif len(val) < 1: raise ConfigException(key, "Name ought to be at least one character") elif any(map(lambda x: x not in allowed_chars, val)): - raise ConfigException(key, "Please use only characters, digits, any of the following: + - # _") + raise ConfigException( + key, + "Please use only characters, digits, any of the following: + - # _", + ) if key == "device.desc": allowed_chars = string.ascii_letters + string.digits + "+-#_ " @@ -184,7 +222,10 @@ def valid(self, key, val): elif len(val) < 1: raise ConfigException(key, "Name ought to be at least one character") elif any(map(lambda x: x not in allowed_chars, val)): - raise ConfigException(key, "Please use only characters, digits, spaces and any of the following: + - # _") + raise ConfigException( + key, + "Please use only characters, digits, spaces and any of the following: + - # _", + ) return True @@ -196,13 +237,14 @@ def saveSettings(self): os.rename(cfg, cfg + ".bak") with open(cfg, "w") as f: - json.dump(self.__config, f, sort_keys=True, indent=4, separators=(',', ': ')) + json.dump( + self.__config, f, sort_keys=True, indent=4, separators=(",", ": ") + ) except Exception as e: print("[-] Failed to save config file %s" % e) raise ConfigException("config", "%s" % e) - def __repr__(self): return self.__config.__repr__() @@ -210,14 +252,16 @@ def __str__(self): return self.__config.__str__() def toDict(self): - """ Return all settings as a dict """ + """Return all settings as a dict""" return self.__config def toJSON(self): """ JSON representation of config """ - return json.dumps(self.__config, sort_keys=True, indent=4, separators=(',', ': ')) + return json.dumps( + self.__config, sort_keys=True, indent=4, separators=(",", ": ") + ) class ConfigException(Exception): @@ -233,4 +277,5 @@ def __str__(self): def __repr__(self): return "<%s %s (%s)>" % (self.__class__.__name__, self.key, self.msg) + config = Config() diff --git a/opencanary/honeycred.py b/opencanary/honeycred.py index 834e0cec..e8704430 100644 --- a/opencanary/honeycred.py +++ b/opencanary/honeycred.py @@ -3,11 +3,15 @@ __all__ = ["buildHoneyCredHook", "cryptcontext"] -cryptcontext = CryptContext(schemes=["pbkdf2_sha512","bcrypt", "sha512_crypt", "plaintext"]) +cryptcontext = CryptContext( + schemes=["pbkdf2_sha512", "bcrypt", "sha512_crypt", "plaintext"] +) + def buildHoneyCredHook(creds): return functools.partial(testManyCreds, creds) + def testCred(cred, username=None, password=None): """ Test if given credentials matches specified credentials @@ -21,13 +25,14 @@ def testCred(cred, username=None, password=None): user_match = True if cred_username is not None: - user_match = (cred_username.encode() == username) + user_match = cred_username.encode() == username password_match = True if cred_password is not None: password_match = cryptcontext.verify(password, cred_password) - return (user_match and password_match) + return user_match and password_match + def testManyCreds(creds, username=None, password=None): for c in creds: diff --git a/opencanary/iphelper.py b/opencanary/iphelper.py index 79488020..9e873ea4 100644 --- a/opencanary/iphelper.py +++ b/opencanary/iphelper.py @@ -1,6 +1,7 @@ import struct import socket + def ip2int(addr): """ Convert an IP in string format to decimal format @@ -8,6 +9,7 @@ def ip2int(addr): return struct.unpack("!I", socket.inet_aton(addr))[0] + def check_ip(ip, network_range): """ Test if the IP is in range @@ -16,7 +18,7 @@ def check_ip(ip, network_range): given /32 is used. It return True if the IP is in the range. """ - netItem = str(network_range).split('/') + netItem = str(network_range).split("/") rangeIP = netItem[0] if len(netItem) == 2: rangeMask = int(netItem[1]) @@ -26,8 +28,8 @@ def check_ip(ip, network_range): try: ripInt = ip2int(rangeIP) ipInt = ip2int(ip) - result = not ((ipInt ^ ripInt) & 0xFFFFFFFF << (32 - rangeMask)); - except: + result = not ((ipInt ^ ripInt) & 0xFFFFFFFF << (32 - rangeMask)) + except: # noqa: E722 result = False return result diff --git a/opencanary/logger.py b/opencanary/logger.py index 86f21eaa..942095e1 100644 --- a/opencanary/logger.py +++ b/opencanary/logger.py @@ -10,23 +10,26 @@ from twisted.internet import reactor import requests -from opencanary.iphelper import * +from opencanary.iphelper import check_ip + class Singleton(type): _instances = {} + def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) return cls._instances[cls] + def getLogger(config): try: - d = config.getVal('logger') - except Exception as e: + d = config.getVal("logger") + except Exception: print("Error: config does not have 'logger' section", file=sys.stderr) exit(1) - classname = d.get('class', None) + classname = d.get("class", None) if classname is None: print("Logger section is missing the class key.", file=sys.stderr) exit(1) @@ -36,7 +39,7 @@ def getLogger(config): print("Logger class (%s) is not defined." % classname, file=sys.stderr) exit(1) - kwargs = d.get('kwargs', None) + kwargs = d.get("kwargs", None) if kwargs is None: print("Logger section is missing the kwargs key.", file=sys.stderr) exit(1) @@ -49,82 +52,85 @@ def getLogger(config): return logger + class LoggerBase(object): - LOG_BASE_BOOT = 1000 - LOG_BASE_MSG = 1001 - LOG_BASE_DEBUG = 1002 - LOG_BASE_ERROR = 1003 - LOG_BASE_PING = 1004 - LOG_BASE_CONFIG_SAVE = 1005 - LOG_BASE_EXAMPLE = 1006 - LOG_FTP_LOGIN_ATTEMPT = 2000 - LOG_HTTP_GET = 3000 - LOG_HTTP_POST_LOGIN_ATTEMPT = 3001 - LOG_SSH_NEW_CONNECTION = 4000 - LOG_SSH_REMOTE_VERSION_SENT = 4001 - LOG_SSH_LOGIN_ATTEMPT = 4002 - LOG_SMB_FILE_OPEN = 5000 - LOG_PORT_SYN = 5001 - LOG_PORT_NMAPOS = 5002 - LOG_PORT_NMAPNULL = 5003 - LOG_PORT_NMAPXMAS = 5004 - LOG_PORT_NMAPFIN = 5005 - LOG_TELNET_LOGIN_ATTEMPT = 6001 - LOG_HTTPPROXY_LOGIN_ATTEMPT = 7001 - LOG_MYSQL_LOGIN_ATTEMPT = 8001 - LOG_MSSQL_LOGIN_SQLAUTH = 9001 - LOG_MSSQL_LOGIN_WINAUTH = 9002 - LOG_TFTP = 10001 - LOG_NTP_MONLIST = 11001 - LOG_VNC = 12001 - LOG_SNMP_CMD = 13001 - LOG_RDP = 14001 - LOG_SIP_REQUEST = 15001 - LOG_GIT_CLONE_REQUEST = 16001 - LOG_REDIS_COMMAND = 17001 - LOG_TCP_BANNER_CONNECTION_MADE = 18001 - LOG_TCP_BANNER_KEEP_ALIVE_CONNECTION_MADE = 18002 - LOG_TCP_BANNER_KEEP_ALIVE_SECRET_RECEIVED = 18003 - LOG_TCP_BANNER_KEEP_ALIVE_DATA_RECEIVED = 18004 - LOG_TCP_BANNER_DATA_RECEIVED = 18005 - LOG_USER_0 = 99000 - LOG_USER_1 = 99001 - LOG_USER_2 = 99002 - LOG_USER_3 = 99003 - LOG_USER_4 = 99004 - LOG_USER_5 = 99005 - LOG_USER_6 = 99006 - LOG_USER_7 = 99007 - LOG_USER_8 = 99008 - LOG_USER_9 = 99009 + LOG_BASE_BOOT = 1000 + LOG_BASE_MSG = 1001 + LOG_BASE_DEBUG = 1002 + LOG_BASE_ERROR = 1003 + LOG_BASE_PING = 1004 + LOG_BASE_CONFIG_SAVE = 1005 + LOG_BASE_EXAMPLE = 1006 + LOG_FTP_LOGIN_ATTEMPT = 2000 + LOG_HTTP_GET = 3000 + LOG_HTTP_POST_LOGIN_ATTEMPT = 3001 + LOG_SSH_NEW_CONNECTION = 4000 + LOG_SSH_REMOTE_VERSION_SENT = 4001 + LOG_SSH_LOGIN_ATTEMPT = 4002 + LOG_SMB_FILE_OPEN = 5000 + LOG_PORT_SYN = 5001 + LOG_PORT_NMAPOS = 5002 + LOG_PORT_NMAPNULL = 5003 + LOG_PORT_NMAPXMAS = 5004 + LOG_PORT_NMAPFIN = 5005 + LOG_TELNET_LOGIN_ATTEMPT = 6001 + LOG_HTTPPROXY_LOGIN_ATTEMPT = 7001 + LOG_MYSQL_LOGIN_ATTEMPT = 8001 + LOG_MSSQL_LOGIN_SQLAUTH = 9001 + LOG_MSSQL_LOGIN_WINAUTH = 9002 + LOG_TFTP = 10001 + LOG_NTP_MONLIST = 11001 + LOG_VNC = 12001 + LOG_SNMP_CMD = 13001 + LOG_RDP = 14001 + LOG_SIP_REQUEST = 15001 + LOG_GIT_CLONE_REQUEST = 16001 + LOG_REDIS_COMMAND = 17001 + LOG_TCP_BANNER_CONNECTION_MADE = 18001 + LOG_TCP_BANNER_KEEP_ALIVE_CONNECTION_MADE = 18002 + LOG_TCP_BANNER_KEEP_ALIVE_SECRET_RECEIVED = 18003 + LOG_TCP_BANNER_KEEP_ALIVE_DATA_RECEIVED = 18004 + LOG_TCP_BANNER_DATA_RECEIVED = 18005 + LOG_USER_0 = 99000 + LOG_USER_1 = 99001 + LOG_USER_2 = 99002 + LOG_USER_3 = 99003 + LOG_USER_4 = 99004 + LOG_USER_5 = 99005 + LOG_USER_6 = 99006 + LOG_USER_7 = 99007 + LOG_USER_8 = 99008 + LOG_USER_9 = 99009 def sanitizeLog(self, logdata): - logdata['node_id'] = self.node_id - logdata['local_time'] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f") - logdata['utc_time'] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f") - logdata['local_time_adjusted'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") - if 'src_host' not in logdata: - logdata['src_host'] = '' - if 'src_port' not in logdata: - logdata['src_port'] = -1 - if 'dst_host' not in logdata: - logdata['dst_host'] = '' - if 'dst_port' not in logdata: - logdata['dst_port'] = -1 - if 'logtype' not in logdata: - logdata['logtype'] = self.LOG_BASE_MSG - if 'logdata' not in logdata: - logdata['logdata'] = {} + logdata["node_id"] = self.node_id + logdata["local_time"] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f") + logdata["utc_time"] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f") + logdata["local_time_adjusted"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") + if "src_host" not in logdata: + logdata["src_host"] = "" + if "src_port" not in logdata: + logdata["src_port"] = -1 + if "dst_host" not in logdata: + logdata["dst_host"] = "" + if "dst_port" not in logdata: + logdata["dst_port"] = -1 + if "logtype" not in logdata: + logdata["logtype"] = self.LOG_BASE_MSG + if "logdata" not in logdata: + logdata["logdata"] = {} return logdata + class PyLogger(LoggerBase): """ Generic python logging """ + __metaclass__ = Singleton def __init__(self, config, handlers, formatters={}): - self.node_id = config.getVal('device.node_id') + self.node_id = config.getVal("device.node_id") # Build config dict to initialise # Ensure all handlers don't drop logs based on severity level @@ -133,14 +139,10 @@ def __init__(self, config, handlers, formatters={}): logconfig = { "version": 1, - "formatters" : formatters, + "formatters": formatters, "handlers": handlers, # initialise all defined logger handlers - "loggers": { - self.node_id : { - "handlers": handlers.keys() - } - } + "loggers": {self.node_id: {"handlers": handlers.keys()}}, } try: @@ -152,14 +154,14 @@ def __init__(self, config, handlers, formatters={}): exit(1) # Check if ignorelist is populated - self.ip_ignorelist = config.getVal('ip.ignorelist', default=[]) - self.logtype_ignorelist = config.getVal('logtype.ignorelist', default=[]) + self.ip_ignorelist = config.getVal("ip.ignorelist", default=[]) + self.logtype_ignorelist = config.getVal("logtype.ignorelist", default=[]) self.logger = logging.getLogger(self.node_id) def error(self, data): - data['local_time'] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f") - msg = '[ERR] %r' % json.dumps(data, sort_keys=True) + data["local_time"] = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f") + msg = "[ERR] %r" % json.dumps(data, sort_keys=True) print(msg, file=sys.stderr) self.logger.warn(msg) @@ -167,23 +169,24 @@ def log(self, logdata, retry=True): logdata = self.sanitizeLog(logdata) # Log only if not in ignorelist notify = True - if 'src_host' in logdata: + if "src_host" in logdata: for ip in self.ip_ignorelist: - if check_ip(logdata['src_host'], ip) == True: + if check_ip(logdata["src_host"], ip) is True: notify = False break - if 'logtype' in logdata and logdata['logtype'] in self.logtype_ignorelist: + if "logtype" in logdata and logdata["logtype"] in self.logtype_ignorelist: notify = False - if notify == True: + if notify is True: self.logger.warn(json.dumps(logdata, sort_keys=True)) + class SocketJSONHandler(SocketHandler): """Emits JSON messages over TCP delimited by newlines ('\n')""" def makeSocket(self, timeout=1): - s = SocketHandler.makeSocket(self,timeout) + s = SocketHandler.makeSocket(self, timeout) s.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) return s @@ -221,50 +224,55 @@ def makePickle(self, record): class HpfeedsHandler(logging.Handler): - def __init__(self,host,port,ident, secret,channels): + def __init__(self, host, port, ident, secret, channels): logging.Handler.__init__(self) - self.host=str(host) - self.port=int(port) - self.ident=str(ident) - self.secret=str(secret) - self.channels=map(str,channels) - hpc=hpfeeds.new(self.host, self.port, self.ident, self.secret) + self.host = str(host) + self.port = int(port) + self.ident = str(ident) + self.secret = str(secret) + self.channels = map(str, channels) + hpc = hpfeeds.new(self.host, self.port, self.ident, self.secret) hpc.subscribe(channels) - self.hpc=hpc + self.hpc = hpc def emit(self, record): try: msg = self.format(record) - self.hpc.publish(self.channels,msg) - except: + self.hpc.publish(self.channels, msg) + except: # noqa: E722 print("Error on publishing to server") + class SlackHandler(logging.Handler): - def __init__(self,webhook_url): + def __init__(self, webhook_url): logging.Handler.__init__(self) - self.webhook_url=webhook_url + self.webhook_url = webhook_url def generate_msg(self, alert): msg = {} - msg['pretext'] = "OpenCanary Alert" - data=json.loads(alert.msg) - msg['fields']=[] - for k,v in data.items(): - msg['fields'].append({'title':k, 'value':json.dumps(v) if type(v) is dict else v}) - return {'attachments':[msg]} + msg["pretext"] = "OpenCanary Alert" + data = json.loads(alert.msg) + msg["fields"] = [] + for k, v in data.items(): + msg["fields"].append( + {"title": k, "value": json.dumps(v) if type(v) is dict else v} + ) + return {"attachments": [msg]} def emit(self, record): data = self.generate_msg(record) - response = requests.post( - self.webhook_url, json=data - ) + response = requests.post(self.webhook_url, json=data) if response.status_code != 200: - print("Error %s sending Slack message, the response was:\n%s" % (response.status_code, response.text)) + print( + "Error %s sending Slack message, the response was:\n%s" + % (response.status_code, response.text) + ) + class TeamsHandler(logging.Handler): - def __init__(self,webhook_url): + def __init__(self, webhook_url): logging.Handler.__init__(self) - self.webhook_url=webhook_url + self.webhook_url = webhook_url def message(self, data): message = { @@ -273,16 +281,14 @@ def message(self, data): "themeColor": "49c176", "summary": "OpenCanary Notification", "title": "OpenCanary Alert", - "sections": [{ - "facts": self.facts(data) - }] + "sections": [{"facts": self.facts(data)}], } return message def facts(self, data, prefix=None): facts = [] for k, v in data.items(): - key = str(k).lower() if prefix is None else prefix + '__' + str(k).lower() + key = str(k).lower() if prefix is None else prefix + "__" + str(k).lower() if type(v) is not dict: facts.append({"name": key, "value": str(v)}) else: @@ -293,10 +299,13 @@ def facts(self, data, prefix=None): def emit(self, record): data = json.loads(record.msg) payload = self.message(data) - headers = {'Content-Type': 'application/json'} + headers = {"Content-Type": "application/json"} response = requests.post(self.webhook_url, headers=headers, json=payload) if response.status_code != 200: - print("Error %s sending Teams message, the response was:\n%s" % (response.status_code, response.text)) + print( + "Error %s sending Teams message, the response was:\n%s" + % (response.status_code, response.text) + ) def map_string(data, mapping): @@ -316,12 +325,14 @@ def map_string(data, mapping): if isinstance(data, (list, set, tuple)): return [map_string(d, mapping) for d in data] if isinstance(data, (str, bytes)): - return (data % mapping) + return data % mapping return data class WebhookHandler(logging.Handler): - def __init__(self, url, method="POST", data=None, status_code=200, ignore=None, **kwargs): + def __init__( + self, url, method="POST", data=None, status_code=200, ignore=None, **kwargs + ): logging.Handler.__init__(self) self.url = url self.method = method @@ -348,9 +359,16 @@ def emit(self, record): data = map_string(deepcopy(data), mapping) if "application/json" in self.kwargs.get("headers", {}).values(): - response = requests.request(method=self.method, url=self.url, json=data, **self.kwargs) + response = requests.request( + method=self.method, url=self.url, json=data, **self.kwargs + ) else: - response = requests.request(method=self.method, url=self.url, data=data, **self.kwargs) + response = requests.request( + method=self.method, url=self.url, data=data, **self.kwargs + ) if response.status_code != self.status_code: - print("Error %s sending Requests payload, the response was:\n%s" % (response.status_code, response.text)) + print( + "Error %s sending Requests payload, the response was:\n%s" + % (response.status_code, response.text) + ) diff --git a/opencanary/modules/__init__.py b/opencanary/modules/__init__.py index 993a01cd..d952efa1 100644 --- a/opencanary/modules/__init__.py +++ b/opencanary/modules/__init__.py @@ -6,29 +6,35 @@ from twisted.internet.protocol import Factory from twisted.internet.protocol import DatagramProtocol -from opencanary.honeycred import * +from opencanary.honeycred import buildHoneyCredHook # Monkey-patch-replace Twisted Protocol with CanaryProtocol class from twisted.internet import protocol + class CanaryProtocol(protocol.Protocol): """TCP protocols (ie. descedents of this class) gain a log method that be can be called with just the event data, as transport data is added here""" def log(self, *args, **kwargs): - if hasattr(self, 'factory') and hasattr(self.factory, 'log'): - kwargs['transport'] = self.transport + if hasattr(self, "factory") and hasattr(self.factory, "log"): + kwargs["transport"] = self.transport return self.factory.log(*args, **kwargs) - raise AttributeError("""Instance of %s does not have 'factory' attribute - or factory does not have a log function.""" % self.__class__.__name__ ) + raise AttributeError( + """Instance of %s does not have 'factory' attribute + or factory does not have a log function.""" + % self.__class__.__name__ + ) + protocol.Protocol = CanaryProtocol + class CanaryService(object): - NAME = 'baseservice' + NAME = "baseservice" - def __init__(self,config=None, logger=None): + def __init__(self, config=None, logger=None): self.config = config self.logger = logger self.logtype = None @@ -56,31 +62,28 @@ def log(self, logdata, **kwargs): For brevity, protocols may pass in Twisted transport argument for logger to get the IPs and ports of the connection. """ - data = { - 'logtype' : self.logtype, - 'logdata' : logdata - } + data = {"logtype": self.logtype, "logdata": logdata} - logtype = kwargs.pop('logtype', None) + logtype = kwargs.pop("logtype", None) if logtype: msg = """Passing in the logtype to log is deprecated. (In future each module will have only only logtype.)""" warnings.warn(msg, DeprecationWarning) - data['logtype'] = logtype + data["logtype"] = logtype - transport = kwargs.pop('transport', None) + transport = kwargs.pop("transport", None) if transport: us = transport.getHost() peer = transport.getPeer() - data['src_host'] = peer.host - data['src_port'] = peer.port - data['dst_host'] = us.host - data['dst_port'] = us.port + data["src_host"] = peer.host + data["src_port"] = peer.port + data["dst_host"] = us.host + data["dst_port"] = us.port # otherwise the module can include IPs and ports as kwargs data.update(kwargs) # run pre-log hooks - if getattr(self ,"honeyCredHook", None): + if getattr(self, "honeyCredHook", None): username = logdata.get("USERNAME", None) password = logdata.get("PASSWORD", None) if username or password: @@ -100,22 +103,21 @@ def getService(self): elif isinstance(self, DatagramProtocol): return internet.UDPServer(self.port, self) - err = 'The class %s does not inherit from either Factory or DatagramProtocol.' % ( - self.__class__.__name__ - ) + err = ( + "The class %s does not inherit from either Factory or DatagramProtocol." + % (self.__class__.__name__) + ) raise Exception(err) -if sys.platform.startswith("linux"): +if sys.platform.startswith("linux"): # noqa: C901 from twisted.python import filepath from twisted.internet import inotify from twisted.python._inotify import INotifyError from twisted.internet.inotify import IN_CREATE - import datetime import os class FileSystemWatcher(object): - def __init__(self, fileName=None): self.path = fileName self.log_dir = os.path.dirname(os.path.realpath(self.path)) @@ -129,7 +131,7 @@ def reopenFiles(self, skipToEnd=True): self.f = open(self.path) if skipToEnd: self.f.seek(0, 2) - except IOError as e: + except IOError: self.f = None self.notifier.startReading() @@ -139,11 +141,15 @@ def reopenFiles(self, skipToEnd=True): pass try: - self.notifier.watch(filepath.FilePath(self.path), - callbacks=[self.onChange]) + self.notifier.watch( + filepath.FilePath(self.path), callbacks=[self.onChange] + ) except INotifyError: - self.notifier.watch(filepath.FilePath(self.log_dir), mask=IN_CREATE, - callbacks=[self.onDirChange]) + self.notifier.watch( + filepath.FilePath(self.log_dir), + mask=IN_CREATE, + callbacks=[self.onDirChange], + ) def start(self): self.notifier = inotify.INotify() @@ -152,26 +158,26 @@ def start(self): def handleLines(self, lines=None): pass - def processAuditLines(self,): + def processAuditLines( + self, + ): if not self.f: return - lines = self.f.read().strip().split('\n') + lines = self.f.read().strip().split("\n") self.handleLines(lines=lines) - def onChange(self, watch, path, mask): - #print path, 'changed', mask # or do something else! + # print path, 'changed', mask # or do something else! if mask != 2: self.reopenFiles() self.processAuditLines() - def onDirChange(self, watch, path, mask): - #print path, ' dir changed', mask # or do something else! - #import pdb; pdb.set_trace() + # print path, ' dir changed', mask # or do something else! + # import pdb; pdb.set_trace() try: self.notifier.ignore(filepath.FilePath(self.log_dir)) except KeyError: diff --git a/opencanary/modules/data/http/skin/nasLogin/static/css/xtheme-gray.css b/opencanary/modules/data/http/skin/nasLogin/static/css/xtheme-gray.css index 8dc9c2a5..919e7329 100644 --- a/opencanary/modules/data/http/skin/nasLogin/static/css/xtheme-gray.css +++ b/opencanary/modules/data/http/skin/nasLogin/static/css/xtheme-gray.css @@ -48,7 +48,7 @@ } /* -.x-color-palette em:hover, .x-color-palette span:hover{ +.x-color-palette em:hover, .x-color-palette span:hover{ background-color: #eaeaea; } */ @@ -421,8 +421,8 @@ ul.x-tab-strip-bottom{ background-image:url(../images/default/button/s-arrow-noline.gif); } -.x-toolbar .x-btn-over .x-btn-mc em.x-btn-split, .x-toolbar .x-btn-click .x-btn-mc em.x-btn-split, -.x-toolbar .x-btn-menu-active .x-btn-mc em.x-btn-split, .x-toolbar .x-btn-pressed .x-btn-mc em.x-btn-split +.x-toolbar .x-btn-over .x-btn-mc em.x-btn-split, .x-toolbar .x-btn-click .x-btn-mc em.x-btn-split, +.x-toolbar .x-btn-menu-active .x-btn-mc em.x-btn-split, .x-toolbar .x-btn-pressed .x-btn-mc em.x-btn-split { background-image:url(../images/gray/button/s-arrow-o.gif); } @@ -431,8 +431,8 @@ ul.x-tab-strip-bottom{ background-image:url(../images/default/button/s-arrow-b-noline.gif); } -.x-toolbar .x-btn-over .x-btn-mc em.x-btn-split-bottom, .x-toolbar .x-btn-click .x-btn-mc em.x-btn-split-bottom, -.x-toolbar .x-btn-menu-active .x-btn-mc em.x-btn-split-bottom, .x-toolbar .x-btn-pressed .x-btn-mc em.x-btn-split-bottom +.x-toolbar .x-btn-over .x-btn-mc em.x-btn-split-bottom, .x-toolbar .x-btn-click .x-btn-mc em.x-btn-split-bottom, +.x-toolbar .x-btn-menu-active .x-btn-mc em.x-btn-split-bottom, .x-toolbar .x-btn-pressed .x-btn-mc em.x-btn-split-bottom { background-image:url(../images/gray/button/s-arrow-bo.gif); } @@ -1348,7 +1348,7 @@ a.x-menu-item { .x-panel-header { color:#333; - font-weight:bold; + font-weight:bold; font-size: 11px; font-family: tahoma,arial,verdana,sans-serif; border-color:#d0d0d0; diff --git a/opencanary/modules/data/http/skin/nasLogin/static/js/misc.js b/opencanary/modules/data/http/skin/nasLogin/static/js/misc.js index 8b30d12f..d602a0e4 100644 --- a/opencanary/modules/data/http/skin/nasLogin/static/js/misc.js +++ b/opencanary/modules/data/http/skin/nasLogin/static/js/misc.js @@ -51,4 +51,3 @@ document.getElementById('ext-comp-1008').innerHTML = wkday + ", " + month + " " updateTime(); window.setInterval(updateTime, 2000); - diff --git a/opencanary/modules/data/httpproxy/skin/squid/auth.html b/opencanary/modules/data/httpproxy/skin/squid/auth.html index 02dcf2ac..930aaa9c 100644 --- a/opencanary/modules/data/httpproxy/skin/squid/auth.html +++ b/opencanary/modules/data/httpproxy/skin/squid/auth.html @@ -2,7 +2,7 @@ ERROR: Cache Access Denied -