From 36c10028a28c96e8a3b7efe2081c4ce128fce18e Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:12:02 +0100 Subject: [PATCH 01/17] Rename bin/certcheck.py -> bin/certcheck --- bin/{certcheck.py => certcheck} | 19 +++++++++++++++++-- setup.py | 2 +- test/moduletests/certcheck/test_certcheck.py | 6 +++--- 3 files changed, 21 insertions(+), 6 deletions(-) rename bin/{certcheck.py => certcheck} (97%) diff --git a/bin/certcheck.py b/bin/certcheck similarity index 97% rename from bin/certcheck.py rename to bin/certcheck index 2a988c8..a2d0fb0 100755 --- a/bin/certcheck.py +++ b/bin/certcheck @@ -1,4 +1,19 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2013 Spotify AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + #Make it a bit more like python3: from __future__ import division @@ -452,12 +467,12 @@ def release(cls): def parse_command_line(): parser = argparse.ArgumentParser( description='Simple certificate expiration check', - epilog="Author: prozlach@spotify.com", + epilog="Author: vespian a t wp.pl", add_help=True,) parser.add_argument( '--version', action='version', - version='1.0') + version='0.3.0') parser.add_argument( "-c", "--config-file", action='store', diff --git a/setup.py b/setup.py index d454f9f..01df247 100755 --- a/setup.py +++ b/setup.py @@ -8,4 +8,4 @@ author_email='prozlach@spotify.com', description='Simplified certificate check', packages=['certcheck'], - scripts=['bin/certcheck.py']) + scripts=['bin/certcheck']) diff --git a/test/moduletests/certcheck/test_certcheck.py b/test/moduletests/certcheck/test_certcheck.py index 005b25a..7306f4a 100644 --- a/test/moduletests/certcheck/test_certcheck.py +++ b/test/moduletests/certcheck/test_certcheck.py @@ -301,7 +301,7 @@ def test_command_line_parsing(self, SysExitMock): old_args = sys.argv #General parsing: - sys.argv = ['./certcheck.py', '-v', '-s', '-d', '-c', './certcheck.json'] + sys.argv = ['./certcheck', '-v', '-s', '-d', '-c', './certcheck.json'] parsed_cmdline = certcheck.parse_command_line() self.assertEqual(parsed_cmdline, {'std_err': True, 'config_file': './certcheck.json', @@ -310,14 +310,14 @@ def test_command_line_parsing(self, SysExitMock): }) #Config file should be a mandatory argument: - sys.argv = ['./certcheck.py', ] + sys.argv = ['./certcheck', ] # Suppres warnings from argparse with mock.patch('sys.stderr'): parsed_cmdline = certcheck.parse_command_line() SysExitMock.assert_called_once_with(2) #Test default values: - sys.argv = ['./certcheck.py', '-c', './certcheck.json'] + sys.argv = ['./certcheck', '-c', './certcheck.json'] parsed_cmdline = certcheck.parse_command_line() self.assertEqual(parsed_cmdline, {'std_err': False, 'config_file': './certcheck.json', From 7d29091c46f47227971535b2927ed9a4b1bf2182 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:19:20 +0100 Subject: [PATCH 02/17] Add gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc From 9e197729f014565e96754fcb6f62d68b2c49c196 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:20:16 +0100 Subject: [PATCH 03/17] Documentation refactoring, merging README.md and USAGE.md --- README.md | 156 ++++++++++++++++++++++++++++++++++++++++++++++++-- doc/README.md | 50 ---------------- doc/USAGE.md | 103 --------------------------------- 3 files changed, 151 insertions(+), 158 deletions(-) delete mode 100644 doc/README.md delete mode 100644 doc/USAGE.md diff --git a/README.md b/README.md index e73e833..22317de 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,152 @@ -Copyright (c) 2013 Spotify AB -Certcheck - Simple certificate checker -Documentation for this project should be found in the doc directory. +# _Certchecker_ -Currently this is a dump of an internal tool; YMMV trying to get this to work -outside Spotify. +_Certchecker is a certificate expiration check capable of scanning GIT repos +and sending data on expiring/expired certificates back to the monitoring system +(currently only Riemann)._ + +## Project Setup + +In order to run certchecker you need to following dependencies installed: +- Bernhard - Riemann client library (https://github.com/banjiewen/bernhard) +- Google's protobuf library +- yaml bindings for python (http://pyyaml.org/) +- Dulwich - python implementation of GIT (https://www.samba.org/~jelmer/dulwich/docs/) +- ssh command in your PATH +- argparse library + +You can also use debian packaging rules from debian/ directory to build a deb +package. + +## Usage + +### Configuration + +Actions taken by the script are determined by its command line and the +configuration file. The command line has a build-in help system: + +``` +usage: certcheck [-h] [--version] -c CONFIG_FILE [-v] [-s] [-d] + +Simple certificate expiration check + +optional arguments: + -h, --help show this help message and exit + --version show program's version number and exit + -c CONFIG_FILE, --config-file CONFIG_FILE + Location of the configuration file + -v, --verbose Provide extra logging messages. + -s, --std-err Log to stderr instead of syslog + -d, --dont-send Do not send data to Riemann [use for debugging] + +Author: vespian a t wp.pl +``` + +The configuration file is a plain YAML document. It's syntax is as follows: + +``` +--- +lockfile: /tmp/certcheck.lock +warn_treshold: 30 +critical_treshold: 15 +riemann_hosts: + static: + - 192.168.122.16:5555:udp + - 192.168.122.16:5555:tcp + by_srv: + - _riemann._tcp + - _riemann._udp +riemann_tags: + - production + - class::certcheck +repo_host: git.example.net +repo_port: 22 +repo_url: /example-repo +repo_masterbranch: refs/heads/production +repo_localdir: /tmp/certcheck-temprepo +repo_user: certcheck +repo_pubkey: ./certcheck_id_rsa + # format - dict, hash as a key, and value as a comment + # sha1sum ./certificate_to_be_ignored +ignored_certs: + aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: "some VPN key" + bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb: "some unused certificate" +``` + +### Operation + +The script connects to the $repo_user@$repo_host:$repo_port via SSH and clones +repository $repo_url to a *bare* repository in "$repo_tmpdir/repository". If +the repository already exists, it is only updated with newest referances. Only +$repo_masterbranch branch is pulled in along with all the objects it points to, +topic branches are not downloaded. + +The connection is established using the $repo_pubkey pubkey, and the $repo_user +itself should have very limited privileges. + +Next, the repository is scanned in search of files ending with one of the +certcheck:CERTIFICATE_EXTENSIONS extensions. Currently all possible +certificate extensions are listed but only ['pem', 'crt', 'cer'] are currently +supported (see certcheck:get_cert_expiration method). For the remaing ones +only a warning is issued. + +For each certificate found a sha1sum is computed, and if the result is found in +$ignored_certs hash, then the certificate is ignored even if it expires/exp- +ired. + +If the number of days till the certificate expires is less than $critical_tresh +(by default 15) - a "critical" partial status is generated, if it less than +$warn_tresh but more than $critical_tresh - a "warning" partial status is gene- +rated. Unsuported certificate yields an 'unknown' state and expired ones of +course the 'critical'. + +All the 'partial status' updates are agregated and each message can only ele- +vate up the final status of the metric send to Riemann. Currently, the hierar- +chy is as follows: + + (lowest)ok->warn->critical->unknown(highest) + +script errors, exceptions and unexcpected conditions result in imidiate elevation +to 'unknown' status and sending the metric to monitoring system ASAP if only +possible. + +IP addresses/ports of the Riemann instances can be defined in two ways: + * statically, by providing a list of riemann instances in $riemann_servers + var. The format of the list entry is hostname:port:proto. 'proto' can be one + of 'udp' or 'tcp'. + * by providing a SRV record, i.e. '_riemann._udp'. All the values + (host, port) will be resolved automatically. Protocol is chosen basing on + the SRV entry itself. + +The final metric is send to *all* Riemann instances with TTL equal to +certcheck:DATA_TTL == 25 hours. + +### Maintenance + +In order to not to let the "$repo_tmpdir/repository" repository grow endlessly +a 'git gc' command should be executed once a day by i.e. a cronjob. It should +repack all the packs and remove dangling objects. +Please see the doc/USAGE.md file for details. + +## Contributing + +All patches are welcome ! Please use Github issue tracking and/or create a pull +request. + +### Testing + +Currenlty the unittest python library is used to perform all the testing. In +test/ directory you can find: +- modules/ - modules used by unittests +- moduletests/ - the unittests themselves +- fabric/ - sample input files and test certificates temporary directories +- output_coverage_html/ - coverage tests results in a form of an html webpage +- test.py - script to start all the unittests + +All the dependencies required for performing the unittests are decribed in debian +packaging scripts and are as follows: +- unittests2 +- coverage +- python-mock +- openssl command in the PATH + +Plus all the dependencies mentioned in 'Project Setup' section. diff --git a/doc/README.md b/doc/README.md deleted file mode 100644 index 421e489..0000000 --- a/doc/README.md +++ /dev/null @@ -1,50 +0,0 @@ -# _Certchecker_ - -_Certchecker is a certificate expiration check capable of scanning GIT repos -and sending data on expiring/expired certificates back to the monitoring system -(currently only Riemann)._ - -## Project Setup - -In order to run certchecker you need to following dependencies installed: -- Bernhard - Riemann client library (https://github.com/banjiewen/bernhard) -- Google's protobuf library -- yaml bindings for python (http://pyyaml.org/) -- Dulwich - python implementation of GIT (https://www.samba.org/~jelmer/dulwich/docs/) -- ssh command in your PATH -- argparse library - -You can also use debian packaging rules from debian/ directory to build a deb -package. - -## Testing - -Currenlty the unittest python library is used to perform all the testing. In -test/ directory you can find: -- modules/ - modules used by unittests -- moduletests/ - the unittests themselves -- fabric/ - sample input files and test certificates temporary directories -- output_coverage_html/ - coverage tests results in a form of an html webpage -- test.py - script to start all the unittests - -All the dependencies required for performing the unittests are decribed in debian -packaging scripts and are as follows: -- unittests2 -- coverage -- python-mock -- openssl command in the PATH - -Plus all the dependencies mentioned in 'Project Setup' section. - -## Usage - -Please see the doc/USAGE.md file for details. - -## Contributing - -All patches are welcome ! Please use Github issue tracking and/or create a pull -request. - -## License - -FIXME - Put the licence here diff --git a/doc/USAGE.md b/doc/USAGE.md deleted file mode 100644 index 6e5c809..0000000 --- a/doc/USAGE.md +++ /dev/null @@ -1,103 +0,0 @@ -## Configuration - -Actions taken by the script are determined by its command line and the -configuration file. The command line has a build-in help system: - - -usage: certcheck.py [-h] [--version] -c CONFIG_FILE [-v] [-s] [-d] - -Simple certificate expiration check - -optional arguments: - -h, --help show this help message and exit - --version show program's version number and exit - -c CONFIG_FILE, --config-file CONFIG_FILE - Location of the configuration file - -v, --verbose Provide extra logging messages. - -s, --std-err Log to stderr instead of syslog - -d, --dont-send Do not send data to Riemann [use for debugging] - -Author: pawel.rozlach a t gmail.com - -The configuration file is a plain YAML document. It's syntax is as follows: - ---- -lockfile: /tmp/certcheck.lock -warn_treshold: 30 -critical_treshold: 15 -riemann_hosts: - static: - - 192.168.122.16:5555:udp - - 192.168.122.16:5555:tcp - by_srv: - - _riemann._tcp - - _riemann._udp -riemann_tags: - - production - - class::certcheck -repo_host: git.example.net -repo_port: 22 -repo_url: /example-repo -repo_masterbranch: refs/heads/production -repo_localdir: /tmp/certcheck-temprepo -repo_user: certcheck -repo_pubkey: ./certcheck_id_rsa -# format - dict, hash as a key, and value as a comment -# sha1sum ./certificate_to_be_ignored -ignored_certs: - aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: "some VPN key" - bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb: "some unused certificate" - -## Operation - -The script connects to the $repo_user@$repo_host:$repo_port via SSH and clones -repository $repo_url to a *bare* repository in "$repo_tmpdir/repository". If -the repository already exists, it is only updated with newest referances. Only -$repo_masterbranch branch is pulled in along with all the objects it points to, -topic branches are not downloaded. - -The connection is established using the $repo_pubkey pubkey, and the $repo_user -itself should have very limited privileges. - -Next, the repository is scanned in search of files ending with one of the -certcheck.py:CERTIFICATE_EXTENSIONS extensions. Currently all possible -certificate extensions are listed but only ['pem', 'crt', 'cer'] are currently -supported (see certcheck.py:get_cert_expiration method). For the remaing ones -only a warning is issued. - -For each certificate found a sha1sum is computed, and if the result is found in -$ignored_certs hash, then the certificate is ignored even if it expires/exp- -ired. - -If the number of days till the certificate expires is less than $critical_tresh -(by default 15) - a "critical" partial status is generated, if it less than -$warn_tresh but more than $critical_tresh - a "warning" partial status is gene- -rated. Unsuported certificate yields an 'unknown' state and expired ones of -course the 'critical'. - -All the 'partial status' updates are agregated and each message can only ele- -vate up the final status of the metric send to Riemann. Currently, the hierar- -chy is as follows: - - (lowest)ok->warn->critical->unknown(highest) - -script errors, exceptions and unexcpected conditions result in imidiate elevation -to 'unknown' status and sending the metric to Riemann ASAP if only possible. - -IP addresses/ports of the Riemann instances can be defined in two ways: - * statically, by providing a list of riemann instances in $riemann_servers - var. The format of the list entry is hostname:port:proto. 'proto' can be one - of 'udp' or 'tcp'. - * by providing a SRV record, i.e. '_riemann._udp'. All the values - (host, port) will be resolved automatically. Protocol is chosen basing on - the SRV entry itself. - -The final metric is send to *all* Riemann instances with TTL equal to -certcheck.py:DATA_TTL == 25 hours. - - -=== Maintenance - -In order to not to let the "$repo_tmpdir/repository" repository grow endlessly -a 'git gc' command should be executed once a day by i.e. a cronjob. It should -repack all the packs and remove dangling objects. From febf16b00d4cdc6a009ed526bc64a05191575f44 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:20:59 +0100 Subject: [PATCH 04/17] Add Apache 2.0 license preamble --- setup.py | 15 +++++++++++++++ test/modules/file_paths.py | 15 +++++++++++++++ test/moduletests/certcheck/test_certcheck.py | 15 +++++++++++++++ test/test.py | 15 +++++++++++++++ 4 files changed, 60 insertions(+) diff --git a/setup.py b/setup.py index 01df247..42939af 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,19 @@ #! /usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2013 Spotify AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + from setuptools import setup diff --git a/test/modules/file_paths.py b/test/modules/file_paths.py index b6f0a9a..719b316 100644 --- a/test/modules/file_paths.py +++ b/test/modules/file_paths.py @@ -1,4 +1,19 @@ #!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# Copyright (c) 2013 Spotify AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + import os.path as op diff --git a/test/moduletests/certcheck/test_certcheck.py b/test/moduletests/certcheck/test_certcheck.py index 7306f4a..e1bf7ae 100644 --- a/test/moduletests/certcheck/test_certcheck.py +++ b/test/moduletests/certcheck/test_certcheck.py @@ -1,4 +1,19 @@ #!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# Copyright (c) 2013 Spotify AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + #Make it a bit more like python3: from __future__ import absolute_import diff --git a/test/test.py b/test/test.py index 6fc65aa..81bab15 100755 --- a/test/test.py +++ b/test/test.py @@ -1,4 +1,19 @@ #!/usr/bin/python -tt +# -*- coding: utf-8 -*- +# Copyright (c) 2013 Spotify AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + #Make it a bit more like python3: from __future__ import absolute_import From 906fca1632d99013ca8f9ecd84cd2b98fc82e7ed Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:22:20 +0100 Subject: [PATCH 05/17] Change package category --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index f0f82ad..74d793c 100644 --- a/debian/control +++ b/debian/control @@ -1,5 +1,5 @@ Source: certcheck -Section: non-free/net +Section: utils Priority: extra Maintainer: Pawel Rozlach Build-Depends: python (>= 2.6.6-3~), debhelper (>= 8), From 075252fac893e01c09bd4a2768819e33eae36558 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:23:14 +0100 Subject: [PATCH 06/17] Version bump, versioning cleanup, maintainer update --- debian/changelog | 8 ++++++++ debian/control | 2 +- setup.py | 6 +++--- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/debian/changelog b/debian/changelog index 0a6c4a3..1298e41 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,11 @@ +certcheck (0.3.0) stable; urgency=low + + * Documentation refactoring + * Make unittests nosetest compatible + * splitting script into modules + + -- Vespian Mon, 18 Sep 2013 14:33:43 +0000 + certcheck (0.2.0) stable; urgency=low * Git integration. diff --git a/debian/control b/debian/control index 74d793c..be71f3c 100644 --- a/debian/control +++ b/debian/control @@ -1,7 +1,7 @@ Source: certcheck Section: utils Priority: extra -Maintainer: Pawel Rozlach +Maintainer: Vespian Build-Depends: python (>= 2.6.6-3~), debhelper (>= 8), python-coverage, openssl, python-openssl, python-bernhard, python-argparse, python-protobuf, python-unittest2, python-yaml, python-dulwich (>= 0.8), diff --git a/setup.py b/setup.py index 42939af..c3e45f5 100755 --- a/setup.py +++ b/setup.py @@ -18,9 +18,9 @@ from setuptools import setup setup(name='certcheck', - version='1', - author=u'Pawel Rozlach', - author_email='prozlach@spotify.com', + version='0.3.0', + author=u'Vespian', + author_email='vespian a t wp.pl', description='Simplified certificate check', packages=['certcheck'], scripts=['bin/certcheck']) From 1f3971ddfc0731638ce64686f38b409326f14f34 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:15:04 +0000 Subject: [PATCH 07/17] Fix program description --- bin/certcheck | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/certcheck b/bin/certcheck index a2d0fb0..818c8ce 100755 --- a/bin/certcheck +++ b/bin/certcheck @@ -466,7 +466,7 @@ class ScriptLock(object): def parse_command_line(): parser = argparse.ArgumentParser( - description='Simple certificate expiration check', + description='Certificate checking tool', epilog="Author: vespian a t wp.pl", add_help=True,) parser.add_argument( diff --git a/setup.py b/setup.py index c3e45f5..45b6e25 100755 --- a/setup.py +++ b/setup.py @@ -21,6 +21,6 @@ version='0.3.0', author=u'Vespian', author_email='vespian a t wp.pl', - description='Simplified certificate check', + description='Certificate checking tool', packages=['certcheck'], scripts=['bin/certcheck']) From fe6fef0ac1c3bfc34130026fcf93d9ffddb45bc0 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 14:37:57 +0000 Subject: [PATCH 08/17] Add missing python-dnspython dependency --- debian/control | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debian/control b/debian/control index be71f3c..5d96e18 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: certcheck Section: utils Priority: extra Maintainer: Vespian -Build-Depends: python (>= 2.6.6-3~), debhelper (>= 8), +Build-Depends: python (>= 2.6.6-3~), debhelper (>= 8), python-dnspython python-coverage, openssl, python-openssl, python-bernhard, python-argparse, python-protobuf, python-unittest2, python-yaml, python-dulwich (>= 0.8), openssh-client @@ -13,5 +13,6 @@ Package: certcheck Version: 0.2.0 Architecture: any Depends: ${python:Depends}, python-openssl, python-bernhard, python-argparse, - python-protobuf, python-yaml, python-dulwich (>= 0.8), openssh-client + python-protobuf, python-yaml, python-dulwich (>= 0.8), openssh-client, + python-dnspython Description: Simple certificate check From 2a0a65cdc20d3a54803259fdc1319f39ec84b8aa Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 15:11:37 +0000 Subject: [PATCH 09/17] Move script body to a separate module --- bin/__init__.py | 0 bin/certcheck | 659 +----------------- certcheck/__init__.py | 671 +++++++++++++++++++ test/moduletests/certcheck/test_certcheck.py | 1 - 4 files changed, 674 insertions(+), 657 deletions(-) delete mode 100644 bin/__init__.py create mode 100755 certcheck/__init__.py diff --git a/bin/__init__.py b/bin/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bin/certcheck b/bin/certcheck index 818c8ce..707f8c7 100755 --- a/bin/certcheck +++ b/bin/certcheck @@ -14,663 +14,10 @@ # License for the specific language governing permissions and limitations under # the License. +import certcheck -#Make it a bit more like python3: -from __future__ import division -from __future__ import nested_scopes -from __future__ import print_function -from __future__ import with_statement - -#Imports: -from OpenSSL.crypto import FILETYPE_PEM -from OpenSSL.crypto import load_certificate -from collections import namedtuple -from datetime import datetime, timedelta -from dulwich.client import SSHGitClient, SubprocessWrapper, TraditionalGitClient -from dulwich.protocol import Protocol -from dulwich.repo import Repo -import argparse -import bernhard -import dns.resolver -import fcntl -import hashlib -import logging -import logging.handlers as lh -import os -import re -import socket -import subprocess -import sys -import yaml - -#Constants: -LOCKFILE_LOCATION = './'+os.path.basename(__file__)+'.lock' -CONFIGFILE_LOCATION = './'+os.path.basename(__file__)+'.conf' -DATA_TTL = 25*60*60 # Data gathered by the script run is valid for 25 hours. -SERVICE_NAME = 'certcheck' -CERTIFICATE_EXTENSIONS = ['der', 'crt', 'pem', 'cer', 'p12', 'pfx', ] - - -class RecoverableException(Exception): - """ - Exception used to differentiate between errors which should be reported - to Riemann, and the ones that should be only logged due to their severity - """ - pass - - -class PubkeySSHGitClient(SSHGitClient): - def __init__(self, host, pubkey, port=None, username=None, *args, **kwargs): - self.host = host - self.port = port - self.pubkey = pubkey - self.username = username - TraditionalGitClient.__init__(self, *args, **kwargs) - self.alternative_paths = {} - - def _connect(self, cmd, path): - #FIXME: This has no way to deal with passphrases.. - #FIXME: can we rely on ssh being in PATH here ? - args = ['ssh', '-x', '-oStrictHostKeyChecking=no'] - args.extend(['-i', self.pubkey]) - if self.port is not None: - args.extend(['-p', str(self.port)]) - if self.username is not None: - host = '{0}@{1}'.format(self.username, self.host) - else: - host = self.host - args.append(host) - args.extend(["{0} '{1}'".format(self._get_cmd_path(cmd), path)]) - proc = subprocess.Popen(args, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - con = SubprocessWrapper(proc) - logging.info("Connected to repo {0}:{1} via ssh".format(self.host, - self.port if self.port else 22)) - return (Protocol(con.read, - con.write, - report_activity=self._report_activity - ), - con.can_read) - - -class LocalMirrorRepo(Repo): - def lookup_files(self, determine_wants, root_sha=None, repo_path=''): - file_list = [] - if root_sha is None: - commit = self.get_object(self.head()) - root_sha = commit.tree - root = self.get_object(root_sha) - if repo_path: - #Extreme verbosity - #logging.debug("Scanning repo directory {0}".format(repo_path)) - pass - else: - logging.info("Scanning repo root directory") - - for item in root.iteritems(): - full_path = os.path.join(repo_path, item.path) - if item.mode & 0b0100000000000000: - #A directory: - subentries = self.lookup_files(determine_wants=determine_wants, - root_sha=item.sha, - repo_path=full_path) - file_list.extend(subentries) - if item.mode & 0b1000000000000000: - #A file, lets check if user wants it: - if determine_wants(item.path): - logging.info("Matching file found: {0}".format(full_path)) - buf = namedtuple("FileTuple", ['path', 'sha']) - buf.path = full_path - buf.content = self.get_object(item.sha).data - file_list.append(buf) - return file_list - - -class CertStore(object): - _remote = None - _local = None - - @classmethod - def initialize(cls, host, port, pubkey, username, repo_localdir, repo_url, - repo_masterbranch): - if cls._remote is None: - cls._remote = PubkeySSHGitClient(host=host, - pubkey=pubkey, - username=username, - port=port, - thin_packs=False, # Not supported by - # dulwich properly - ) - if not os.path.exists(os.path.join(repo_localdir, "objects")): - if not os.path.exists(repo_localdir): - os.mkdir(repo_localdir, 0700) - cls._local = LocalMirrorRepo.init_bare(repo_localdir) - else: - cls._local = LocalMirrorRepo(repo_localdir) - - #We are only interested in 'production' branch, not the topic branches - #all the commits linked to the master will be downloaded as well of - #course - def wants_master_only(refs): - return [sha for (ref, sha) in refs.iteritems() - if ref == repo_masterbranch] - refs = cls._remote.fetch(path=repo_url, target=cls._local, - determine_wants=wants_master_only) - cls._local["HEAD"] = refs[repo_masterbranch] - - @classmethod - def lookup_certs(cls, cert_suffixes): - if cls._local is None: - raise RecoverableException("Local repo mirror has not been " + - "initialized yet") - - def wants_all_certs(path): - if len(path) >= 5 and path[-4] == '.' and \ - path[-3:] in cert_suffixes: - return True - else: - return False - certs = cls._local.lookup_files(determine_wants=wants_all_certs) - logging.info("{0} certificates found".format(len(certs))) - return certs - - -class ScriptConfiguration(object): - - _config = dict() - - @classmethod - def load_config(cls, file_path): - """ - @param string file_path path to the configuration file - """ - try: - with open(file_path, 'r') as fh: - cls._config = yaml.load(fh) - except IOError as e: - logging.error("Failed to open config file {0}: {1}".format( - file_path, e)) - sys.exit(1) - except (yaml.parser.ParserError, ValueError) as e: - logging.error("File {0} is not a proper yaml document: {1}".format( - file_path, e)) - sys.exit(1) - - @classmethod - def get_val(cls, key): - return cls._config[key] - - -class ScriptStatus(object): - - _STATES = {'ok': 0, - 'warn': 1, - 'critical': 2, - 'unknown': 3, - } - - _exit_status = 'ok' - _exit_message = '' - _riemann_connections = [] - _riemann_tags = None - _hostname = '' - _debug = None - - @classmethod - def _send_data(cls, event): - for riemann_connection in cls._riemann_connections: - logging.info('Sending event {0}, '.format(str(event)) + - 'using Riemann conn {0}:{1}'.format( - riemann_connection.host, riemann_connection.port) - ) - if not cls._debug: - try: - riemann_connection.send(event) - except Exception as e: - logging.exception("Failed to send event to Rieman host: " + - "{0}".format(str(e)) - ) - continue - else: - logging.info("Event sent succesfully") - else: - logging.info('Debug flag set, I am performing no-op instead of ' - 'real sent call') - - @classmethod - def _name2ip(cls, name): - if re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', name): - #IP entry: - return name - else: - #Hostname, we need to resolve it: - try: - ipaddr = dns.resolver.query(name, 'A') - except dns.resolver.NXDOMAIN: - logging.error("A record for {0} was not found".format(name)) - return name # Let somebody else worry about it ;) - - return ipaddr[0].to_text() - - @classmethod - def _resolve_srv_hosts(cls, name): - result = [] - logging.debug("Resolving " + name) - if name.find('._udp') > 0: - proto = 'udp' - elif name.find('._tcp') > 0: - proto = 'tcp' - else: - raise RecoverableException("Entry {0} ".format(name) + - "is not a valid SRV name") - - try: - resolved = dns.resolver.query(name, 'SRV') - except dns.resolver.NXDOMAIN: - logging.error("Entry {0} does not exist, skipping.") - return [] - - for rdata in resolved: - entry = namedtuple("RiemannHost", ['host', 'port', 'proto']) - entry.host = cls._name2ip(rdata.target.to_text()) - if entry.host is None: - continue - entry.port = rdata.port - entry.proto = proto - result.append(entry) - logging.debug("String {0} resolved as {1}".format(name, str(entry))) - - return result - - @classmethod - def _resolve_static_entry(cls, name): - entry = namedtuple("RiemannHost", ['host', 'port', 'proto']) - try: - a, b, c = name.split(":") - entry.host = cls._name2ip(a) - if entry.host is None: - raise ValueError() - entry.port = int(b) # Raises ValueError by itself - if c in ['tcp', 'udp']: - entry.proto = c - else: - raise ValueError() - except ValueError: - logging.error("String {0} is not a valid ip:port:proto entry") - return [] - - logging.debug("String {0} resolved as {1}".format(name, str(entry))) - return [entry] - - @classmethod - def initialize(cls, riemann_hosts_config, riemann_tags, debug=False): - cls._riemann_tags = riemann_tags - cls._hostname = socket.gethostname() - cls._debug = debug - cls._exit_status = 'ok' - cls._exit_message = '' - cls._riemann_connections = [] # FIXME - we should probably do - # some disconect here if we re-initialize - # probably using conn.shutdown() call - - if not riemann_tags: - logging.error('there should be at least one Riemann tag defined.') - return # should it sys.exit or just return ?? - tmp = [] - if "static" in riemann_hosts_config: - for line in riemann_hosts_config["static"]: - tmp.extend(cls._resolve_static_entry(line)) - - if "by_srv" in riemann_hosts_config: - for line in riemann_hosts_config["by_srv"]: - tmp.extend(cls._resolve_srv_hosts(line)) - - for riemann_host in tmp: - try: - if riemann_host.proto == 'tcp': - riemann_connection = bernhard.Client(riemann_host.host, - riemann_host.port, - bernhard.TCPTransport) - elif riemann_host.proto == 'udp': - riemann_connection = bernhard.Client(riemann_host.host, - riemann_host.port, - bernhard.UDPTransport) - else: - logging.error("Unsupported transport {0}".format(riemann_host.proto) + - ", not connected to {1}".format(riemann_host)) - except Exception as e: - logging.exception("Failed to connect to Rieman host " + - "{0}: {1}, ".format(riemann_host, str(e)) + - "address has been exluded from the list.") - continue - - logging.debug("Connected to Riemann instance {0}".format(riemann_host)) - cls._riemann_connections.append(riemann_connection) - - if not cls._riemann_connections: - logging.error("There are no active connections to Riemann, " + - "metrics will not be send!") - - @classmethod - def notify_immediate(cls, exit_status, exit_message): - """ - Imediatelly send given data to Riemann - """ - if exit_status not in cls._STATES: - logging.error("Trying to issue an immediate notification" + - "with malformed exit_status: " + exit_status) - return - - if not exit_message: - logging.error("Trying to issue an immediate" + - "notification without any message") - return - - logging.warn("notify_immediate, " + - "exit_status=<{0}>, exit_message=<{1}>".format( - exit_status, exit_message)) - event = { - 'host': cls._hostname, - 'service': SERVICE_NAME, - 'state': exit_status, - 'description': exit_message, - 'tags': cls._riemann_tags, - 'ttl': DATA_TTL, - } - - cls._send_data(event) - - @classmethod - def notify_agregated(cls): - """ - Send all agregated data to Riemann - """ - - if cls._exit_status == 'ok' and cls._exit_message == '': - cls._exit_message = 'All certificates are OK' - - logging.debug("notify_agregated, " + - "exit_status=<{0}>, exit_message=<{1}>".format( - cls._exit_status, cls._exit_message)) - - event = { - 'host': cls._hostname, - 'service': SERVICE_NAME, - 'state': cls._exit_status, - 'description': cls._exit_message, - 'tags': cls._riemann_tags, - 'ttl': DATA_TTL, - } - - cls._send_data(event) - - @classmethod - def update(cls, exit_status, exit_message): - """ - Accumullate a small bit of data in class fields - """ - if exit_status not in cls._STATES: - logging.error("Trying to do the status update" + - "with malformed exit_status: " + exit_status) - return - - logging.info("updating script status, " + - "exit_status=<{0}>, exit_message=<{1}>".format( - exit_status, exit_message)) - if cls._STATES[cls._exit_status] < cls._STATES[exit_status]: - cls._exit_status = exit_status - # ^ we only escalate up... - if exit_message: - if cls._exit_message: - cls._exit_message += '\n' - cls._exit_message += exit_message - - -class ScriptLock(object): - #python lockfile is brain-damaged, we have to write our own class :/ - _fh = None - _file_path = None - - @classmethod - def init(cls, file_path): - cls._file_path = file_path - - @classmethod - def aqquire(cls): - if cls._fh: - logging.warn("File lock already aquired") - return - try: - cls._fh = open(cls._file_path, 'w') - #flock is nice because it is automatically released when the - #process dies/terminates - fcntl.flock(cls._fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except IOError: - if cls._fh: - cls._fh.close() - raise RecoverableException("{0} ".format(cls._file_path) + - "is already locked by a different " + - "process or cannot be created.") - cls._fh.write(str(os.getpid())) - cls._fh.flush() - - @classmethod - def release(cls): - if not cls._fh: - raise RecoverableException("Trying to release non-existant lock") - cls._fh.close() - cls._fh = None - os.unlink(cls._file_path) - - -def parse_command_line(): - parser = argparse.ArgumentParser( - description='Certificate checking tool', - epilog="Author: vespian a t wp.pl", - add_help=True,) - parser.add_argument( - '--version', - action='version', - version='0.3.0') - parser.add_argument( - "-c", "--config-file", - action='store', - required=True, - help="Location of the configuration file") - parser.add_argument( - "-v", "--verbose", - action='store_true', - required=False, - help="Provide extra logging messages.") - parser.add_argument( - "-s", "--std-err", - action='store_true', - required=False, - help="Log to stderr instead of syslog") - parser.add_argument( - "-d", "--dont-send", - action='store_true', - required=False, - help="Do not send data to Riemann [use for debugging]") - - args = parser.parse_args() - return {'std_err': args.std_err, - 'verbose': args.verbose, - 'config_file': args.config_file, - 'dont_send': args.dont_send, - } - - -def get_cert_expiration(certificate, ignored_certs): - if certificate.path[-3:] in ['pem', 'crt', 'cer']: - try: - #Many bad things can happen here, but still - we can recover! :) - cert_hash = hashlib.sha1(certificate.content).hexdigest() - if cert_hash in ignored_certs: - #This cert should be ignored - logging.info("certificate {0} (sha1sum: {1})".format( - certificate.path, cert_hash) + " has been ignored.") - return None - #Workaround for -----BEGIN TRUSTED CERTIFICATE----- - if certificate.content.find('TRUSTED ') > -1: - logging.info("'TRUSTED' string has been removed from " + - "certificate {0} (sha1sum: {1})".format( - certificate.path, cert_hash)) - certificate.content = certificate.content.replace('TRUSTED ', - '') - cert_data = load_certificate(FILETYPE_PEM, certificate.content) - expiry_date = cert_data.get_notAfter() - #Return datetime object: - return datetime.strptime(expiry_date, '%Y%m%d%H%M%SZ') - except Exception as e: - msg = "Script cannot parse certificate {0}: {1}".format( - certificate.path, str(e)) - logging.warn(msg) - ScriptStatus.update('unknown', msg) - return None - else: - ScriptStatus.update('unknown', - "Certificate {0} is of unsupported type, ".format( - certificate.path) + - "the script cannot check the expiry date.") - return None - - -def main(config_file, std_err=False, verbose=True, dont_send=False): - try: - #Configure logging: - fmt = logging.Formatter('%(filename)s[%(process)d] %(levelname)s: ' + - '%(message)s') - logger = logging.getLogger() - if verbose: - logger.setLevel(logging.DEBUG) - else: - logger.setLevel(logging.INFO) - if std_err: - handler = logging.StreamHandler() - else: - handler = lh.SysLogHandler(address='/dev/log', - facility=lh.SysLogHandler.LOG_USER) - handler.setFormatter(fmt) - logger.addHandler(handler) - - logger.info("Certcheck is starting, command line arguments:" + - "config_file={0}, ".format(config_file) + - "std_err={0}, ".format(std_err) + - "verbose={0}, ".format(verbose) - ) - - #FIXME - Remember to correctly configure syslog, otherwise rsyslog will - #discard messages - ScriptConfiguration.load_config(config_file) - - logger.debug("Remote repo is is: {0}@{1}:{2}{3}->{4}".format( - ScriptConfiguration.get_val("repo_user"), - ScriptConfiguration.get_val("repo_host"), - ScriptConfiguration.get_val("repo_port"), - ScriptConfiguration.get_val("repo_url"), - ScriptConfiguration.get_val("repo_masterbranch")) + - ", local repository dir is {0}".format( - ScriptConfiguration.get_val('repo_localdir')) + - ", repository key is {0}".format( - ScriptConfiguration.get_val('repo_pubkey')) + - ", warn_thresh is {0}".format( - ScriptConfiguration.get_val('warn_treshold')) + - ", crit_thresh is {0}".format( - ScriptConfiguration.get_val('critical_treshold')) - ) - - #Initialize Riemann reporting: - ScriptStatus.initialize( - riemann_hosts_config=ScriptConfiguration.get_val("riemann_hosts"), - riemann_tags=ScriptConfiguration.get_val("riemann_tags"), - debug=dont_send, - ) - - # verify the configuration - msg = [] - if ScriptConfiguration.get_val('warn_treshold') <= 0: - msg.append('certificate expiration warn threshold should be > 0.') - if ScriptConfiguration.get_val('critical_treshold') <= 0: - msg.append('certificate expiration critical threshold should be > 0.') - if ScriptConfiguration.get_val('critical_treshold') >= \ - ScriptConfiguration.get_val('warn_treshold'): - msg.append('warninig threshold should be greater than critical treshold.') - - #if there are problems with thresholds then there is no point in continuing: - if msg: - ScriptStatus.notify_immediate('unknown', - "Configuration file contains errors: " + - ','.join(msg)) - sys.exit(1) - - #Make sure that we are the only ones running on the server: - ScriptLock.init(ScriptConfiguration.get_val('lockfile')) - ScriptLock.aqquire() - - #Initialize our repo mirror: - CertStore.initialize(host=ScriptConfiguration.get_val("repo_host"), - port=ScriptConfiguration.get_val("repo_port"), - pubkey=ScriptConfiguration.get_val('repo_pubkey'), - username=ScriptConfiguration.get_val("repo_user"), - repo_localdir=ScriptConfiguration.get_val( - 'repo_localdir'), - repo_url=ScriptConfiguration.get_val("repo_url"), - repo_masterbranch=ScriptConfiguration.get_val( - "repo_masterbranch"), - ) - - for cert in CertStore.lookup_certs(CERTIFICATE_EXTENSIONS): - cert_expiration = get_cert_expiration(cert, - ignored_certs=ScriptConfiguration.get_val( - "ignored_certs") - ) - if cert_expiration is None: - continue - # -3 days is in fact -4 days, 23:59:58.817181 - # so we compensate and round up - # additionally, openssl uses utc dates - now = datetime.utcnow() - timedelta(days=1) - time_left = cert_expiration - now # timedelta object - if time_left.days < 0: - ScriptStatus.update('critical', - "Certificate {0} expired {1} days ago.".format( - cert.path, abs(time_left.days))) - elif time_left.days == 0: - ScriptStatus.update('critical', - "Certificate {0} expires today.".format( - cert.path)) - elif time_left.days < ScriptConfiguration.get_val("critical_treshold"): - ScriptStatus.update('critical', - "Certificate {0} is about to expire in {1} days.".format( - cert.path, time_left.days)) - elif time_left.days < ScriptConfiguration.get_val("warn_treshold"): - ScriptStatus.update('warn', - "Certificate {0} is about to expire in {1} days.".format( - cert.path, time_left.days)) - else: - logger.info("{0} expires in {1} days - OK!".format( - cert.path, time_left.days)) - - ScriptStatus.notify_agregated() - ScriptLock.release() - sys.exit(0) - - except RecoverableException as e: - msg = str(e) - logging.critical(msg) - ScriptStatus.notify_immediate('unknown', msg) - sys.exit(1) - except AssertionError as e: - #Unittest require it: - raise - except Exception as e: - msg = "Exception occured: {0}".format(e.__class__.__name__) - logging.exception(msg) - sys.exit(1) if __name__ == '__main__': - args_dict = parse_command_line() + args_dict = certcheck.parse_command_line() - main(**args_dict) + certcheck.main(**args_dict) diff --git a/certcheck/__init__.py b/certcheck/__init__.py new file mode 100755 index 0000000..8bc5569 --- /dev/null +++ b/certcheck/__init__.py @@ -0,0 +1,671 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2013 Spotify AB +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + + +#Make it a bit more like python3: +from __future__ import division +from __future__ import nested_scopes +from __future__ import print_function +from __future__ import with_statement + +#Imports: +from OpenSSL.crypto import FILETYPE_PEM +from OpenSSL.crypto import load_certificate +from collections import namedtuple +from datetime import datetime, timedelta +from dulwich.client import SSHGitClient, SubprocessWrapper, TraditionalGitClient +from dulwich.protocol import Protocol +from dulwich.repo import Repo +import argparse +import bernhard +import dns.resolver +import fcntl +import hashlib +import logging +import logging.handlers as lh +import os +import re +import socket +import subprocess +import sys +import yaml + +#Constants: +LOCKFILE_LOCATION = './'+os.path.basename(__file__)+'.lock' +CONFIGFILE_LOCATION = './'+os.path.basename(__file__)+'.conf' +DATA_TTL = 25*60*60 # Data gathered by the script run is valid for 25 hours. +SERVICE_NAME = 'certcheck' +CERTIFICATE_EXTENSIONS = ['der', 'crt', 'pem', 'cer', 'p12', 'pfx', ] + + +class RecoverableException(Exception): + """ + Exception used to differentiate between errors which should be reported + to Riemann, and the ones that should be only logged due to their severity + """ + pass + + +class PubkeySSHGitClient(SSHGitClient): + def __init__(self, host, pubkey, port=None, username=None, *args, **kwargs): + self.host = host + self.port = port + self.pubkey = pubkey + self.username = username + TraditionalGitClient.__init__(self, *args, **kwargs) + self.alternative_paths = {} + + def _connect(self, cmd, path): + #FIXME: This has no way to deal with passphrases.. + #FIXME: can we rely on ssh being in PATH here ? + args = ['ssh', '-x', '-oStrictHostKeyChecking=no'] + args.extend(['-i', self.pubkey]) + if self.port is not None: + args.extend(['-p', str(self.port)]) + if self.username is not None: + host = '{0}@{1}'.format(self.username, self.host) + else: + host = self.host + args.append(host) + args.extend(["{0} '{1}'".format(self._get_cmd_path(cmd), path)]) + proc = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + con = SubprocessWrapper(proc) + logging.info("Connected to repo {0}:{1} via ssh".format(self.host, + self.port if self.port else 22)) + return (Protocol(con.read, + con.write, + report_activity=self._report_activity + ), + con.can_read) + + +class LocalMirrorRepo(Repo): + def lookup_files(self, determine_wants, root_sha=None, repo_path=''): + file_list = [] + if root_sha is None: + commit = self.get_object(self.head()) + root_sha = commit.tree + root = self.get_object(root_sha) + if repo_path: + #Extreme verbosity + #logging.debug("Scanning repo directory {0}".format(repo_path)) + pass + else: + logging.info("Scanning repo root directory") + + for item in root.iteritems(): + full_path = os.path.join(repo_path, item.path) + if item.mode & 0b0100000000000000: + #A directory: + subentries = self.lookup_files(determine_wants=determine_wants, + root_sha=item.sha, + repo_path=full_path) + file_list.extend(subentries) + if item.mode & 0b1000000000000000: + #A file, lets check if user wants it: + if determine_wants(item.path): + logging.info("Matching file found: {0}".format(full_path)) + buf = namedtuple("FileTuple", ['path', 'sha']) + buf.path = full_path + buf.content = self.get_object(item.sha).data + file_list.append(buf) + return file_list + + +class CertStore(object): + _remote = None + _local = None + + @classmethod + def initialize(cls, host, port, pubkey, username, repo_localdir, repo_url, + repo_masterbranch): + if cls._remote is None: + cls._remote = PubkeySSHGitClient(host=host, + pubkey=pubkey, + username=username, + port=port, + thin_packs=False, # Not supported by + # dulwich properly + ) + if not os.path.exists(os.path.join(repo_localdir, "objects")): + if not os.path.exists(repo_localdir): + os.mkdir(repo_localdir, 0700) + cls._local = LocalMirrorRepo.init_bare(repo_localdir) + else: + cls._local = LocalMirrorRepo(repo_localdir) + + #We are only interested in 'production' branch, not the topic branches + #all the commits linked to the master will be downloaded as well of + #course + def wants_master_only(refs): + return [sha for (ref, sha) in refs.iteritems() + if ref == repo_masterbranch] + refs = cls._remote.fetch(path=repo_url, target=cls._local, + determine_wants=wants_master_only) + cls._local["HEAD"] = refs[repo_masterbranch] + + @classmethod + def lookup_certs(cls, cert_suffixes): + if cls._local is None: + raise RecoverableException("Local repo mirror has not been " + + "initialized yet") + + def wants_all_certs(path): + if len(path) >= 5 and path[-4] == '.' and \ + path[-3:] in cert_suffixes: + return True + else: + return False + certs = cls._local.lookup_files(determine_wants=wants_all_certs) + logging.info("{0} certificates found".format(len(certs))) + return certs + + +class ScriptConfiguration(object): + + _config = dict() + + @classmethod + def load_config(cls, file_path): + """ + @param string file_path path to the configuration file + """ + try: + with open(file_path, 'r') as fh: + cls._config = yaml.load(fh) + except IOError as e: + logging.error("Failed to open config file {0}: {1}".format( + file_path, e)) + sys.exit(1) + except (yaml.parser.ParserError, ValueError) as e: + logging.error("File {0} is not a proper yaml document: {1}".format( + file_path, e)) + sys.exit(1) + + @classmethod + def get_val(cls, key): + return cls._config[key] + + +class ScriptStatus(object): + + _STATES = {'ok': 0, + 'warn': 1, + 'critical': 2, + 'unknown': 3, + } + + _exit_status = 'ok' + _exit_message = '' + _riemann_connections = [] + _riemann_tags = None + _hostname = '' + _debug = None + + @classmethod + def _send_data(cls, event): + for riemann_connection in cls._riemann_connections: + logging.info('Sending event {0}, '.format(str(event)) + + 'using Riemann conn {0}:{1}'.format( + riemann_connection.host, riemann_connection.port) + ) + if not cls._debug: + try: + riemann_connection.send(event) + except Exception as e: + logging.exception("Failed to send event to Rieman host: " + + "{0}".format(str(e)) + ) + continue + else: + logging.info("Event sent succesfully") + else: + logging.info('Debug flag set, I am performing no-op instead of ' + 'real sent call') + + @classmethod + def _name2ip(cls, name): + if re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', name): + #IP entry: + return name + else: + #Hostname, we need to resolve it: + try: + ipaddr = dns.resolver.query(name, 'A') + except dns.resolver.NXDOMAIN: + logging.error("A record for {0} was not found".format(name)) + return name # Let somebody else worry about it ;) + + return ipaddr[0].to_text() + + @classmethod + def _resolve_srv_hosts(cls, name): + result = [] + logging.debug("Resolving " + name) + if name.find('._udp') > 0: + proto = 'udp' + elif name.find('._tcp') > 0: + proto = 'tcp' + else: + raise RecoverableException("Entry {0} ".format(name) + + "is not a valid SRV name") + + try: + resolved = dns.resolver.query(name, 'SRV') + except dns.resolver.NXDOMAIN: + logging.error("Entry {0} does not exist, skipping.") + return [] + + for rdata in resolved: + entry = namedtuple("RiemannHost", ['host', 'port', 'proto']) + entry.host = cls._name2ip(rdata.target.to_text()) + if entry.host is None: + continue + entry.port = rdata.port + entry.proto = proto + result.append(entry) + logging.debug("String {0} resolved as {1}".format(name, str(entry))) + + return result + + @classmethod + def _resolve_static_entry(cls, name): + entry = namedtuple("RiemannHost", ['host', 'port', 'proto']) + try: + a, b, c = name.split(":") + entry.host = cls._name2ip(a) + if entry.host is None: + raise ValueError() + entry.port = int(b) # Raises ValueError by itself + if c in ['tcp', 'udp']: + entry.proto = c + else: + raise ValueError() + except ValueError: + logging.error("String {0} is not a valid ip:port:proto entry") + return [] + + logging.debug("String {0} resolved as {1}".format(name, str(entry))) + return [entry] + + @classmethod + def initialize(cls, riemann_hosts_config, riemann_tags, debug=False): + cls._riemann_tags = riemann_tags + cls._hostname = socket.gethostname() + cls._debug = debug + cls._exit_status = 'ok' + cls._exit_message = '' + cls._riemann_connections = [] # FIXME - we should probably do + # some disconect here if we re-initialize + # probably using conn.shutdown() call + + if not riemann_tags: + logging.error('there should be at least one Riemann tag defined.') + return # should it sys.exit or just return ?? + tmp = [] + if "static" in riemann_hosts_config: + for line in riemann_hosts_config["static"]: + tmp.extend(cls._resolve_static_entry(line)) + + if "by_srv" in riemann_hosts_config: + for line in riemann_hosts_config["by_srv"]: + tmp.extend(cls._resolve_srv_hosts(line)) + + for riemann_host in tmp: + try: + if riemann_host.proto == 'tcp': + riemann_connection = bernhard.Client(riemann_host.host, + riemann_host.port, + bernhard.TCPTransport) + elif riemann_host.proto == 'udp': + riemann_connection = bernhard.Client(riemann_host.host, + riemann_host.port, + bernhard.UDPTransport) + else: + logging.error("Unsupported transport {0}".format(riemann_host.proto) + + ", not connected to {1}".format(riemann_host)) + except Exception as e: + logging.exception("Failed to connect to Rieman host " + + "{0}: {1}, ".format(riemann_host, str(e)) + + "address has been exluded from the list.") + continue + + logging.debug("Connected to Riemann instance {0}".format(riemann_host)) + cls._riemann_connections.append(riemann_connection) + + if not cls._riemann_connections: + logging.error("There are no active connections to Riemann, " + + "metrics will not be send!") + + @classmethod + def notify_immediate(cls, exit_status, exit_message): + """ + Imediatelly send given data to Riemann + """ + if exit_status not in cls._STATES: + logging.error("Trying to issue an immediate notification" + + "with malformed exit_status: " + exit_status) + return + + if not exit_message: + logging.error("Trying to issue an immediate" + + "notification without any message") + return + + logging.warn("notify_immediate, " + + "exit_status=<{0}>, exit_message=<{1}>".format( + exit_status, exit_message)) + event = { + 'host': cls._hostname, + 'service': SERVICE_NAME, + 'state': exit_status, + 'description': exit_message, + 'tags': cls._riemann_tags, + 'ttl': DATA_TTL, + } + + cls._send_data(event) + + @classmethod + def notify_agregated(cls): + """ + Send all agregated data to Riemann + """ + + if cls._exit_status == 'ok' and cls._exit_message == '': + cls._exit_message = 'All certificates are OK' + + logging.debug("notify_agregated, " + + "exit_status=<{0}>, exit_message=<{1}>".format( + cls._exit_status, cls._exit_message)) + + event = { + 'host': cls._hostname, + 'service': SERVICE_NAME, + 'state': cls._exit_status, + 'description': cls._exit_message, + 'tags': cls._riemann_tags, + 'ttl': DATA_TTL, + } + + cls._send_data(event) + + @classmethod + def update(cls, exit_status, exit_message): + """ + Accumullate a small bit of data in class fields + """ + if exit_status not in cls._STATES: + logging.error("Trying to do the status update" + + "with malformed exit_status: " + exit_status) + return + + logging.info("updating script status, " + + "exit_status=<{0}>, exit_message=<{1}>".format( + exit_status, exit_message)) + if cls._STATES[cls._exit_status] < cls._STATES[exit_status]: + cls._exit_status = exit_status + # ^ we only escalate up... + if exit_message: + if cls._exit_message: + cls._exit_message += '\n' + cls._exit_message += exit_message + + +class ScriptLock(object): + #python lockfile is brain-damaged, we have to write our own class :/ + _fh = None + _file_path = None + + @classmethod + def init(cls, file_path): + cls._file_path = file_path + + @classmethod + def aqquire(cls): + if cls._fh: + logging.warn("File lock already aquired") + return + try: + cls._fh = open(cls._file_path, 'w') + #flock is nice because it is automatically released when the + #process dies/terminates + fcntl.flock(cls._fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + if cls._fh: + cls._fh.close() + raise RecoverableException("{0} ".format(cls._file_path) + + "is already locked by a different " + + "process or cannot be created.") + cls._fh.write(str(os.getpid())) + cls._fh.flush() + + @classmethod + def release(cls): + if not cls._fh: + raise RecoverableException("Trying to release non-existant lock") + cls._fh.close() + cls._fh = None + os.unlink(cls._file_path) + + +def parse_command_line(): + parser = argparse.ArgumentParser( + description='Certificate checking tool', + epilog="Author: vespian a t wp.pl", + add_help=True,) + parser.add_argument( + '--version', + action='version', + version='0.3.0') + parser.add_argument( + "-c", "--config-file", + action='store', + required=True, + help="Location of the configuration file") + parser.add_argument( + "-v", "--verbose", + action='store_true', + required=False, + help="Provide extra logging messages.") + parser.add_argument( + "-s", "--std-err", + action='store_true', + required=False, + help="Log to stderr instead of syslog") + parser.add_argument( + "-d", "--dont-send", + action='store_true', + required=False, + help="Do not send data to Riemann [use for debugging]") + + args = parser.parse_args() + return {'std_err': args.std_err, + 'verbose': args.verbose, + 'config_file': args.config_file, + 'dont_send': args.dont_send, + } + + +def get_cert_expiration(certificate, ignored_certs): + if certificate.path[-3:] in ['pem', 'crt', 'cer']: + try: + #Many bad things can happen here, but still - we can recover! :) + cert_hash = hashlib.sha1(certificate.content).hexdigest() + if cert_hash in ignored_certs: + #This cert should be ignored + logging.info("certificate {0} (sha1sum: {1})".format( + certificate.path, cert_hash) + " has been ignored.") + return None + #Workaround for -----BEGIN TRUSTED CERTIFICATE----- + if certificate.content.find('TRUSTED ') > -1: + logging.info("'TRUSTED' string has been removed from " + + "certificate {0} (sha1sum: {1})".format( + certificate.path, cert_hash)) + certificate.content = certificate.content.replace('TRUSTED ', + '') + cert_data = load_certificate(FILETYPE_PEM, certificate.content) + expiry_date = cert_data.get_notAfter() + #Return datetime object: + return datetime.strptime(expiry_date, '%Y%m%d%H%M%SZ') + except Exception as e: + msg = "Script cannot parse certificate {0}: {1}".format( + certificate.path, str(e)) + logging.warn(msg) + ScriptStatus.update('unknown', msg) + return None + else: + ScriptStatus.update('unknown', + "Certificate {0} is of unsupported type, ".format( + certificate.path) + + "the script cannot check the expiry date.") + return None + + +def main(config_file, std_err=False, verbose=True, dont_send=False): + try: + #Configure logging: + fmt = logging.Formatter('%(filename)s[%(process)d] %(levelname)s: ' + + '%(message)s') + logger = logging.getLogger() + if verbose: + logger.setLevel(logging.DEBUG) + else: + logger.setLevel(logging.INFO) + if std_err: + handler = logging.StreamHandler() + else: + handler = lh.SysLogHandler(address='/dev/log', + facility=lh.SysLogHandler.LOG_USER) + handler.setFormatter(fmt) + logger.addHandler(handler) + + logger.info("Certcheck is starting, command line arguments:" + + "config_file={0}, ".format(config_file) + + "std_err={0}, ".format(std_err) + + "verbose={0}, ".format(verbose) + ) + + #FIXME - Remember to correctly configure syslog, otherwise rsyslog will + #discard messages + ScriptConfiguration.load_config(config_file) + + logger.debug("Remote repo is is: {0}@{1}:{2}{3}->{4}".format( + ScriptConfiguration.get_val("repo_user"), + ScriptConfiguration.get_val("repo_host"), + ScriptConfiguration.get_val("repo_port"), + ScriptConfiguration.get_val("repo_url"), + ScriptConfiguration.get_val("repo_masterbranch")) + + ", local repository dir is {0}".format( + ScriptConfiguration.get_val('repo_localdir')) + + ", repository key is {0}".format( + ScriptConfiguration.get_val('repo_pubkey')) + + ", warn_thresh is {0}".format( + ScriptConfiguration.get_val('warn_treshold')) + + ", crit_thresh is {0}".format( + ScriptConfiguration.get_val('critical_treshold')) + ) + + #Initialize Riemann reporting: + ScriptStatus.initialize( + riemann_hosts_config=ScriptConfiguration.get_val("riemann_hosts"), + riemann_tags=ScriptConfiguration.get_val("riemann_tags"), + debug=dont_send, + ) + + # verify the configuration + msg = [] + if ScriptConfiguration.get_val('warn_treshold') <= 0: + msg.append('certificate expiration warn threshold should be > 0.') + if ScriptConfiguration.get_val('critical_treshold') <= 0: + msg.append('certificate expiration critical threshold should be > 0.') + if ScriptConfiguration.get_val('critical_treshold') >= \ + ScriptConfiguration.get_val('warn_treshold'): + msg.append('warninig threshold should be greater than critical treshold.') + + #if there are problems with thresholds then there is no point in continuing: + if msg: + ScriptStatus.notify_immediate('unknown', + "Configuration file contains errors: " + + ','.join(msg)) + sys.exit(1) + + #Make sure that we are the only ones running on the server: + ScriptLock.init(ScriptConfiguration.get_val('lockfile')) + ScriptLock.aqquire() + + #Initialize our repo mirror: + CertStore.initialize(host=ScriptConfiguration.get_val("repo_host"), + port=ScriptConfiguration.get_val("repo_port"), + pubkey=ScriptConfiguration.get_val('repo_pubkey'), + username=ScriptConfiguration.get_val("repo_user"), + repo_localdir=ScriptConfiguration.get_val( + 'repo_localdir'), + repo_url=ScriptConfiguration.get_val("repo_url"), + repo_masterbranch=ScriptConfiguration.get_val( + "repo_masterbranch"), + ) + + for cert in CertStore.lookup_certs(CERTIFICATE_EXTENSIONS): + cert_expiration = get_cert_expiration(cert, + ignored_certs=ScriptConfiguration.get_val( + "ignored_certs") + ) + if cert_expiration is None: + continue + # -3 days is in fact -4 days, 23:59:58.817181 + # so we compensate and round up + # additionally, openssl uses utc dates + now = datetime.utcnow() - timedelta(days=1) + time_left = cert_expiration - now # timedelta object + if time_left.days < 0: + ScriptStatus.update('critical', + "Certificate {0} expired {1} days ago.".format( + cert.path, abs(time_left.days))) + elif time_left.days == 0: + ScriptStatus.update('critical', + "Certificate {0} expires today.".format( + cert.path)) + elif time_left.days < ScriptConfiguration.get_val("critical_treshold"): + ScriptStatus.update('critical', + "Certificate {0} is about to expire in {1} days.".format( + cert.path, time_left.days)) + elif time_left.days < ScriptConfiguration.get_val("warn_treshold"): + ScriptStatus.update('warn', + "Certificate {0} is about to expire in {1} days.".format( + cert.path, time_left.days)) + else: + logger.info("{0} expires in {1} days - OK!".format( + cert.path, time_left.days)) + + ScriptStatus.notify_agregated() + ScriptLock.release() + sys.exit(0) + + except RecoverableException as e: + msg = str(e) + logging.critical(msg) + ScriptStatus.notify_immediate('unknown', msg) + sys.exit(1) + except AssertionError as e: + #Unittest require it: + raise + except Exception as e: + msg = "Exception occured: {0}".format(e.__class__.__name__) + logging.exception(msg) + sys.exit(1) diff --git a/test/moduletests/certcheck/test_certcheck.py b/test/moduletests/certcheck/test_certcheck.py index e1bf7ae..b6072c8 100644 --- a/test/moduletests/certcheck/test_certcheck.py +++ b/test/moduletests/certcheck/test_certcheck.py @@ -39,7 +39,6 @@ #To perform local imports first we need to fix PYTHONPATH: pwd = os.path.abspath(os.path.dirname(__file__)) -sys.path.append(os.path.abspath(pwd + '/../../../bin/')) sys.path.append(os.path.abspath(pwd + '/../../modules/')) #Local imports: From fae72c0e8fc9bda45ad18ac4f9ecb286b37a7270 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 15:13:37 +0000 Subject: [PATCH 10/17] Switch unittests to nosetests, provide simple config --- nose.cfg | 6 ++++++ test/.coveragerc | 27 ------------------------- test/test.py | 52 ------------------------------------------------ 3 files changed, 6 insertions(+), 79 deletions(-) create mode 100644 nose.cfg delete mode 100644 test/.coveragerc delete mode 100755 test/test.py diff --git a/nose.cfg b/nose.cfg new file mode 100644 index 0000000..e13c638 --- /dev/null +++ b/nose.cfg @@ -0,0 +1,6 @@ +[nosetests] +with-coverage=1 +cover-package=certcheck +cover-erase=1 +cover-html=1 +cover-html-dir=test/output_coverage_html/ diff --git a/test/.coveragerc b/test/.coveragerc deleted file mode 100644 index 4a280d0..0000000 --- a/test/.coveragerc +++ /dev/null @@ -1,27 +0,0 @@ -# .coveragerc to control coverage.py -[run] -branch = True -exclude_lines = - # Have to re-enable the standard pragma - pragma: no cover - - # Don't complain about missing debug-only code: - def __repr__ - if self\.debug - - # Don't complain if tests don't hit defensive assertion code: - raise AssertionError - raise NotImplementedError - - # Don't complain if non-runnable code isn't run: - if 0: - if __name__ == .__main__.: -source = - ../bin/ - ../certcheck/ - - -[html] -directory = output_coverage_html - -[paths] diff --git a/test/test.py b/test/test.py deleted file mode 100755 index 81bab15..0000000 --- a/test/test.py +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/python -tt -# -*- coding: utf-8 -*- -# Copyright (c) 2013 Spotify AB -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - - -#Make it a bit more like python3: -from __future__ import absolute_import -from __future__ import print_function - -import coverage -import sys -import unittest - - -def main(): - major, minor, micro, releaselevel, serial = sys.version_info - - if major == 2 and minor < 7: - print("In order to run tests you need at least Python 2.7") - sys.exit(1) - - if major == 3: - print("Tests were not tested on Python 3.X, use at your own risk") - sys.exit(1) - - #Perform coverage analisys: - cov = coverage.coverage() - - cov.start() - #Discover the test and execute them: - loader = unittest.TestLoader() - tests = loader.discover('.') - testRunner = unittest.runner.TextTestRunner(descriptions=True, verbosity=1) - testRunner.run(tests) - cov.stop() - - cov.html_report() - -if __name__ == '__main__': - main() From 0513e4a59eecece3f9faa1ce758d3bbaef302c6e Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 15:13:54 +0000 Subject: [PATCH 11/17] Ignore coverage output files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0d20b64..c9e64a9 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ *.pyc +/.coverage From 08a4fdd633792caf3268579c6a05e63c31f9e64f Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 15:34:10 +0000 Subject: [PATCH 12/17] Fix missing comma --- debian/control | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/control b/debian/control index 5d96e18..be17909 100644 --- a/debian/control +++ b/debian/control @@ -2,7 +2,7 @@ Source: certcheck Section: utils Priority: extra Maintainer: Vespian -Build-Depends: python (>= 2.6.6-3~), debhelper (>= 8), python-dnspython +Build-Depends: python (>= 2.6.6-3~), debhelper (>= 8), python-dnspython, python-coverage, openssl, python-openssl, python-bernhard, python-argparse, python-protobuf, python-unittest2, python-yaml, python-dulwich (>= 0.8), openssh-client From 65e518aa531aa1ac1e8ba47d5efc4d9008476700 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 15:34:56 +0000 Subject: [PATCH 13/17] Add more stuff to .gitignore --- .gitignore | 3 +++ debian/.gitignore | 2 ++ 2 files changed, 5 insertions(+) create mode 100644 debian/.gitignore diff --git a/.gitignore b/.gitignore index c9e64a9..9022dbd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ *.pyc /.coverage +/build* +certcheck.egg-info/* +debian/files diff --git a/debian/.gitignore b/debian/.gitignore new file mode 100644 index 0000000..6f373e7 --- /dev/null +++ b/debian/.gitignore @@ -0,0 +1,2 @@ +certcheck* + From 20e1023bf68c80f86354949cafc71f0978324dcb Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 15:35:44 +0000 Subject: [PATCH 14/17] Add Debian copyright file --- debian/copyright | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 debian/copyright diff --git a/debian/copyright b/debian/copyright new file mode 100644 index 0000000..68e26fd --- /dev/null +++ b/debian/copyright @@ -0,0 +1,8 @@ +Format-Specification: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ + +Files: * +Copyright: 2012-2013 Spotify AB +License: Apache-2.0 + On Debian systems, the full text of the Apache 2.0 license can be found in the + file `/usr/share/common-licenses/Apache-2.0'. + From 880588241d912b79b7b93f57bbebee70229de576 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 15:36:12 +0000 Subject: [PATCH 15/17] Add some additionall info to setup.py --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 45b6e25..98583a8 100755 --- a/setup.py +++ b/setup.py @@ -19,8 +19,10 @@ setup(name='certcheck', version='0.3.0', - author=u'Vespian', + author='Vespian', author_email='vespian a t wp.pl', + license='ASF2.0', + url='https://github.com/vespian/certcheck', description='Certificate checking tool', packages=['certcheck'], scripts=['bin/certcheck']) From 47e80bc5541cac4a0f405eb3b35745c824728100 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 16:19:02 +0000 Subject: [PATCH 16/17] Add more docstrings --- certcheck/__init__.py | 51 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/certcheck/__init__.py b/certcheck/__init__.py index 8bc5569..5e51f47 100755 --- a/certcheck/__init__.py +++ b/certcheck/__init__.py @@ -60,6 +60,11 @@ class RecoverableException(Exception): class PubkeySSHGitClient(SSHGitClient): + """ + Simple class used to add pubkey authentication to the SSHGitClient class. + In the base class it is not supported, and using password authentication + for a script is insecure. + """ def __init__(self, host, pubkey, port=None, username=None, *args, **kwargs): self.host = host self.port = port @@ -96,6 +101,17 @@ def _connect(self, cmd, path): class LocalMirrorRepo(Repo): def lookup_files(self, determine_wants, root_sha=None, repo_path=''): + """ + Search the repo for files described by the determine_wants + function. The function itself operates on the file paths in a repo and + must return True for objects of interest. + + The search is done recursively, with each iteration scanning just one + repo directory. In case a directory is found the root_sha and repo_path + parameters are provided for a next iteration of the function. + + The result is a list of the filenames accumulated by all iterations. + """ file_list = [] if root_sha is None: commit = self.get_object(self.head()) @@ -128,6 +144,10 @@ def lookup_files(self, determine_wants, root_sha=None, repo_path=''): class CertStore(object): + """ + Provides local clone of a remote repo plus some extra functionality to + ease extracting of the certificates from the repository + """ _remote = None _local = None @@ -161,6 +181,11 @@ def wants_master_only(refs): @classmethod def lookup_certs(cls, cert_suffixes): + """ + Find all the certificates in the repository. The classification is made + by checking whether file suffix can be found in th list of certificate + suffixes found in cert_suffixes parameter. + """ if cls._local is None: raise RecoverableException("Local repo mirror has not been " + "initialized yet") @@ -177,7 +202,9 @@ def wants_all_certs(path): class ScriptConfiguration(object): - + """ + Simple file configuration class basing on the YAML format + """ _config = dict() @classmethod @@ -219,6 +246,10 @@ class ScriptStatus(object): @classmethod def _send_data(cls, event): + """ + Send script status to all Riemann servers using all the protocols that + were configured. + """ for riemann_connection in cls._riemann_connections: logging.info('Sending event {0}, '.format(str(event)) + 'using Riemann conn {0}:{1}'.format( @@ -240,6 +271,9 @@ def _send_data(cls, event): @classmethod def _name2ip(cls, name): + """ + Resolve a dns name. In case it is already an IP - just return it. + """ if re.match('\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}', name): #IP entry: return name @@ -255,6 +289,10 @@ def _name2ip(cls, name): @classmethod def _resolve_srv_hosts(cls, name): + """ + Find Riemann servers by resolving SRV record, provide some sanity + checks as well. + """ result = [] logging.debug("Resolving " + name) if name.find('._udp') > 0: @@ -285,6 +323,10 @@ def _resolve_srv_hosts(cls, name): @classmethod def _resolve_static_entry(cls, name): + """ + Find Riemann servers by resolving plain A record, provide some sanity + checks as well. + """ entry = namedtuple("RiemannHost", ['host', 'port', 'proto']) try: a, b, c = name.split(":") @@ -428,7 +470,7 @@ def update(cls, exit_status, exit_message): class ScriptLock(object): - #python lockfile is brain-damaged, we have to write our own class :/ + #python lockfile isn't usefull, we have to write our own class _fh = None _file_path = None @@ -503,6 +545,11 @@ def parse_command_line(): def get_cert_expiration(certificate, ignored_certs): + """ + Extract the certificate expiration date for a certificate blob. Handle + ignored certificates by comparing shasum of the blob with entries in the + ignored_certs list + """ if certificate.path[-3:] in ['pem', 'crt', 'cer']: try: #Many bad things can happen here, but still - we can recover! :) From 957d555205b097c01c0ff58ec50d5e35f6085ba9 Mon Sep 17 00:00:00 2001 From: Vespian Date: Sun, 9 Mar 2014 20:44:54 +0000 Subject: [PATCH 17/17] Fix documentation reg. unittest, add run_tests.py There is a problem with running nosetest - the previous html coverage reports are not properly removed thus subsequent runs of nosetest --with-coverage... result in "File exists error". The old run_tests.py file was restored and put into the main directory. It takes care of removing old html reports and running the tests. --- .coveragerc | 26 ++++++++++++++++++++++++++ README.md | 36 +++++++++++++++++++++++++++++++++--- nose.cfg | 6 ------ run_tests.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 .coveragerc delete mode 100644 nose.cfg create mode 100755 run_tests.py diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..5e7d51a --- /dev/null +++ b/.coveragerc @@ -0,0 +1,26 @@ +# .coveragerc to control coverage.py +[run] +branch = True +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: +source = + ./certcheck/ + + +[html] +directory = test/output_coverage_html + +[paths] diff --git a/README.md b/README.md index 22317de..9ebaf8c 100644 --- a/README.md +++ b/README.md @@ -140,7 +140,38 @@ test/ directory you can find: - moduletests/ - the unittests themselves - fabric/ - sample input files and test certificates temporary directories - output_coverage_html/ - coverage tests results in a form of an html webpage -- test.py - script to start all the unittests + +Unittests can be started either by using *nosetest* command: + +``` +certcheck/ (master✗) # nosetests +[20:33:02] +...... +---------------------------------------------------------------------- +Ran 6 tests in 0.449s + +OK +``` + +or by issuing the *run_tests.py* command: + +``` +certcheck/ (master✗) # run_tests.py +[20:33:04] +Created test certificate expired_3_days.pem +Created test certificate expire_6_days.pem +Created test certificate expire_21_days.pem +Created test certificate expire_41_days.pem +Created test certificate expire_41_days.der +...... +---------------------------------------------------------------------- +Ran 6 tests in 0.362s + +OK +``` + +The difference is that the *run_tests.py* takes care of generating coverage +reports for you. All the dependencies required for performing the unittests are decribed in debian packaging scripts and are as follows: @@ -148,5 +179,4 @@ packaging scripts and are as follows: - coverage - python-mock - openssl command in the PATH - -Plus all the dependencies mentioned in 'Project Setup' section. +, plus all the dependencies mentioned in 'Project Setup' section. diff --git a/nose.cfg b/nose.cfg deleted file mode 100644 index e13c638..0000000 --- a/nose.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[nosetests] -with-coverage=1 -cover-package=certcheck -cover-erase=1 -cover-html=1 -cover-html-dir=test/output_coverage_html/ diff --git a/run_tests.py b/run_tests.py new file mode 100755 index 0000000..8b4d90e --- /dev/null +++ b/run_tests.py @@ -0,0 +1,48 @@ +#!/usr/bin/python -tt + +#Make it a bit more like python3: +from __future__ import absolute_import +from __future__ import print_function + +import coverage +import os +import shutil +import sys +import unittest + + +def main(): + major, minor, micro, releaselevel, serial = sys.version_info + + if major == 2 and minor < 7: + print("In order to run tests you need at least Python 2.7") + sys.exit(1) + + if major == 3: + print("Tests were not tested on Python 3.X, use at your own risk") + sys.exit(1) + + #Cleanup old html report: + for root, dirs, files in os.walk('test/output_coverage_html/'): + for f in files: + if f == '.gitignore' or f == '.empty_dir': + continue + os.unlink(os.path.join(root, f)) + for d in dirs: + shutil.rmtree(os.path.join(root, d)) + + #Perform coverage analisys: + cov = coverage.coverage() + + cov.start() + #Discover the test and execute them: + loader = unittest.TestLoader() + tests = loader.discover('./test/') + testRunner = unittest.runner.TextTestRunner(descriptions=True, verbosity=1) + testRunner.run(tests) + cov.stop() + + cov.html_report() + +if __name__ == '__main__': + main()