format with black and add a pre-commit config

Author: jcjones

.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+- repo: git://github.com/pre-commit/pre-commit-hooks
+  rev: HEAD
+  hooks:
+    - id: check-ast
+    - id: detect-private-key
+    - id: detect-aws-credentials
+    - id: check-merge-conflict
+    - id: end-of-file-fixer
+    - id: requirements-txt-fixer
+    - id: trailing-whitespace
+- repo:  git://github.com/psf/black
+  rev: HEAD
+  hooks:
+    - id: black
+- repo: local
+  hooks:
+    - id: filter-cascade-tests
+      name: Tests for filter-cascade
+      language: system
+      entry: python3 -m unittest filtercascade/test.py
+      pass_filenames: false
+      files: '.py$'

LICENSE

@@ -35,7 +35,7 @@ Mozilla Public License Version 2.0
     means any form of the work other than Source Code Form.
 
 1.7. "Larger Work"
-    means a work that combines Covered Software with other material, in 
+    means a work that combines Covered Software with other material, in
     a separate file or files, that is not Covered Software.
 
 1.8. "License"
@@ -371,4 +371,3 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 
   This Source Code Form is "Incompatible With Secondary Licenses", as
   defined by the Mozilla Public License, v. 2.0.
-

README.md

@@ -1,3 +1,2 @@
 # filter-cascade
 A python filter cascade implementation
-

filtercascade/__init__.py

@@ -12,33 +12,35 @@
 
 log = logging.getLogger(__name__)
 
+
 class HashAlgorithm(IntEnum):
     MURMUR3 = 1
 
+
 # A simple-as-possible bloom filter implementation making use of version 3 of the 32-bit murmur
 # hash function (for compat with multi-level-bloom-filter-js).
 # mgoodwin 2018
 class Bloomer:
-    LAYER_FMT = b'<BIIB'
+    LAYER_FMT = b"<BIIB"
 
     def __init__(self, *, size, nHashFuncs, level, hashAlg=HashAlgorithm.MURMUR3):
         self.nHashFuncs = nHashFuncs
         self.size = size
         self.level = level
         self.hashAlg = hashAlg
 
-        self.bitarray = bitarray.bitarray(self.size, endian='little')
+        self.bitarray = bitarray.bitarray(self.size, endian="little")
         self.bitarray.setall(False)
 
     def hash(self, seed, key):
         if not isinstance(key, bytes):
             to_bytes_op = getattr(key, "to_bytes", None)
             if isinstance(key, str):
-                key = key.encode('utf-8')
+                key = key.encode("utf-8")
             elif callable(to_bytes_op):
                 key = to_bytes_op()
             else:
-                key = str(key).encode('utf-8')
+                key = str(key).encode("utf-8")
 
         if self.hashAlg != HashAlgorithm.MURMUR3:
             raise Exception(f"Unknown hash algorithm: {self.hashAlg}")
@@ -67,7 +69,9 @@ def tofile(self, f):
         """Write the bloom filter to file object `f'. Underlying bits
         are written as machine values. This is much more space
         efficient than pickling the object."""
-        f.write(pack(self.LAYER_FMT, self.hashAlg, self.size, self.nHashFuncs, self.level))
+        f.write(
+            pack(self.LAYER_FMT, self.hashAlg, self.size, self.nHashFuncs, self.level)
+        )
         f.flush()
         self.bitarray.tofile(f)
 
@@ -92,22 +96,34 @@ def from_buf(cls, buf):
         hashAlgInt, size, nHashFuncs, level = unpack(Bloomer.LAYER_FMT, buf[0:10])
         byte_count = math.ceil(size / 8)
         ba = bitarray.bitarray(endian="little")
-        ba.frombytes(buf[10:10 + byte_count])
-        bloomer = Bloomer(size=1, nHashFuncs=nHashFuncs, level=level, hashAlg=HashAlgorithm(hashAlgInt))
+        ba.frombytes(buf[10 : 10 + byte_count])
+        bloomer = Bloomer(
+            size=1,
+            nHashFuncs=nHashFuncs,
+            level=level,
+            hashAlg=HashAlgorithm(hashAlgInt),
+        )
         bloomer.size = size
-        log.debug("Size is {}, level {}, nHashFuncs, {}".format(
-            size, level, nHashFuncs))
+        log.debug(
+            "Size is {}, level {}, nHashFuncs, {}".format(size, level, nHashFuncs)
+        )
         bloomer.bitarray = ba
 
-        return (buf[10 + byte_count:], bloomer)
+        return (buf[10 + byte_count :], bloomer)
 
 
 class FilterCascade:
-    DIFF_FMT = b'<III'
-    VERSION_FMT = b'<H'
-
-    def __init__(self, filters, error_rates=[0.02, 0.5], growth_factor=1.1,
-                 min_filter_length=10000, version=1):
+    DIFF_FMT = b"<III"
+    VERSION_FMT = b"<H"
+
+    def __init__(
+        self,
+        filters,
+        error_rates=[0.02, 0.5],
+        growth_factor=1.1,
+        min_filter_length=10000,
+        version=1,
+    ):
         self.filters = filters
         self.error_rates = error_rates
         self.growth_factor = growth_factor
@@ -149,22 +165,29 @@ def initialize(self, *, include, exclude):
                     Bloomer.filter_with_characteristics(
                         max(
                             int(include_len * self.growth_factor),
-                            self.min_filter_length), er, depth))
+                            self.min_filter_length,
+                        ),
+                        er,
+                        depth,
+                    )
+                )
             else:
                 # Filter already created for this layer. Check size and resize if needed.
                 required_size = Bloomer.calc_size(
-                    self.filters[depth - 1].nHashFuncs, include_len, er)
+                    self.filters[depth - 1].nHashFuncs, include_len, er
+                )
                 if self.filters[depth - 1].size < required_size:
                     # Resize filter
-                    self.filters[depth -
-                                 1] = Bloomer.filter_with_characteristics(
-                                     int(include_len * self.growth_factor),
-                                     er, depth)
+                    self.filters[depth - 1] = Bloomer.filter_with_characteristics(
+                        int(include_len * self.growth_factor), er, depth
+                    )
                     log.info("Resized filter at {}-depth layer".format(depth))
             filter = self.filters[depth - 1]
             log.debug(
-                "Initializing the {}-depth layer. err={} include_len={} size={} hashes={}"
-                .format(depth, er, include_len, filter.size, filter.nHashFuncs))
+                "Initializing the {}-depth layer. err={} include_len={} size={} hashes={}".format(
+                    depth, er, include_len, filter.size, filter.nHashFuncs
+                )
+            )
             # loop over the elements that *should* be there. Add them to the filter.
             for elem in include:
                 filter.add(elem)
@@ -180,22 +203,32 @@ def initialize(self, *, include, exclude):
             endtime = datetime.datetime.utcnow()
             log.debug(
                 "Took {} ms to process layer {} with bit count {}".format(
-                    (endtime - starttime).seconds * 1000 +
-                    (endtime - starttime).microseconds / 1000, depth,
-                    len(filter.bitarray)))
+                    (endtime - starttime).seconds * 1000
+                    + (endtime - starttime).microseconds / 1000,
+                    depth,
+                    len(filter.bitarray),
+                )
+            )
             # Sanity check layer growth.  Bit count should be going down
             # as false positive rate decreases.
             if depth > 2:
                 if len(filter.bitarray) > len(self.filters[depth - 3].bitarray):
                     sequentialGrowthLayers += 1
                     log.warning(
                         "Increase in false positive rate detected. Depth {} has {}"
-                        " bits and depth {} has {} bits. {}/{} allowed warnings."
-                        .format(depth, len(filter.bitarray), depth - 3 + 1,
-                                len(self.filters[depth - 3].bitarray),
-                                sequentialGrowthLayers, maxSequentialGrowthLayers))
+                        " bits and depth {} has {} bits. {}/{} allowed warnings.".format(
+                            depth,
+                            len(filter.bitarray),
+                            depth - 3 + 1,
+                            len(self.filters[depth - 3].bitarray),
+                            sequentialGrowthLayers,
+                            maxSequentialGrowthLayers,
+                        )
+                    )
                     if sequentialGrowthLayers >= maxSequentialGrowthLayers:
-                        log.error("Too many sequential false positive increases detected. Aborting.")
+                        log.error(
+                            "Too many sequential false positive increases detected. Aborting."
+                        )
                         self.filters.clear()
                         return
                 else:
@@ -211,8 +244,9 @@ def initialize(self, *, include, exclude):
             del self.filters[depth:]
 
     def __contains__(self, elem):
-        for layer, filter in [(idx + 1, self.filters[idx])
-                              for idx in range(len(self.filters))]:
+        for layer, filter in [
+            (idx + 1, self.filters[idx]) for idx in range(len(self.filters))
+        ]:
             even = layer % 2 == 0
             if elem in filter:
                 if layer == len(self.filters):
@@ -238,8 +272,10 @@ def layerCount(self):
     def saveDiffMeta(self, f):
         for filter in self.filters:
             f.write(
-                pack(FilterCascade.DIFF_FMT, filter.size, filter.nHashFuncs,
-                     filter.level))
+                pack(
+                    FilterCascade.DIFF_FMT, filter.size, filter.nHashFuncs, filter.level
+                )
+            )
 
     # Follows the bitarray.tofile parameter convention.
     def tofile(self, f):
@@ -255,7 +291,7 @@ def tofile(self, f):
     @classmethod
     def from_buf(cls, buf):
         log.debug(len(buf))
-        (version, ) = unpack(FilterCascade.VERSION_FMT, buf[0:2])
+        (version,) = unpack(FilterCascade.VERSION_FMT, buf[0:2])
         if version != 1:
             raise Exception(f"Unknown version: {version}")
         buf = buf[2:]
@@ -273,15 +309,14 @@ def loadDiffMeta(cls, f):
         size = calcsize(FilterCascade.DIFF_FMT)
         data = f.read()
         while len(data) >= size:
-            filtersize, nHashFuncs, level = unpack(FilterCascade.DIFF_FMT,
-                                                   data[:size])
-            filters.append(
-                Bloomer(size=filtersize, nHashFuncs=nHashFuncs, level=level))
+            filtersize, nHashFuncs, level = unpack(FilterCascade.DIFF_FMT, data[:size])
+            filters.append(Bloomer(size=filtersize, nHashFuncs=nHashFuncs, level=level))
             data = data[size:]
         return FilterCascade(filters)
 
     @classmethod
     def cascade_with_characteristics(cls, capacity, error_rates, layer=0):
         return FilterCascade(
             [Bloomer.filter_with_characteristics(capacity, error_rates[0])],
-            error_rates=error_rates)
+            error_rates=error_rates,
+        )

filtercascade/test.py

@@ -98,8 +98,9 @@ def test_fc_input_formats(self):
     def test_fc_include_not_list(self):
         f = filtercascade.FilterCascade([])
         with self.assertRaises(TypeError):
-            f.initialize(include=predictable_serial_gen(1),
-                         exclude=predictable_serial_gen(1))
+            f.initialize(
+                include=predictable_serial_gen(1), exclude=predictable_serial_gen(1)
+            )
 
     def test_fc_exclude_must_be_iterable(self):
         f = filtercascade.FilterCascade([])
@@ -114,14 +115,13 @@ def test_fc_iterable(self):
         # slice off a set and re-use the remainder
         revocations = set(islice(serials, 3_000))
 
-        f.initialize(include=revocations,
-                     exclude=serials)
+        f.initialize(include=revocations, exclude=serials)
 
         self.assertEqual(len(f.filters), 3)
         self.assertEqual(f.filters[0].size, 81272)
         self.assertEqual(f.filters[1].size, 14400)
         self.assertEqual(f.filters[2].size, 14400)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()

requirements.txt

@@ -1,2 +1,2 @@
 bitarray >= 0.9.2
-mmh3 >= 2.5.1
+mmh3 >= 2.5.1

setup.py

@@ -1,23 +1,22 @@
 from setuptools import setup, find_packages
 
-setup(name='filtercascade',
-      version='0.2.2',
-      description='A simple bloom filter cascade implementation in Python',
-      long_description='A bloom filter cascade implementation in Python using the 32-bit variant of murmurhash3.',
-      classifiers=[
-        'Development Status :: 5 - Production/Stable',
-        'License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)',
-        'Programming Language :: Python :: 3',
-      ],
-      keywords='bloom filter cascade multi level mlbf',
-      url='http://github.com/mozmark/filter-cascade',
-      author='Mark Goodwin',
-      author_email='[email protected]',
-      license='Mozilla Public License 2.0 (MPL 2.0)',
-      packages=['filtercascade'],
-      install_requires=[
-          'bitarray>=0.9.2',
-          'mmh3>=2.5.1',
-      ],
-      include_package_data=True,
-      zip_safe=False)
+setup(
+    name="filtercascade",
+    version="0.2.2",
+    description="A simple bloom filter cascade implementation in Python",
+    long_description="A bloom filter cascade implementation in Python using the 32-bit variant of murmurhash3.",
+    classifiers=[
+        "Development Status :: 5 - Production/Stable",
+        "License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)",
+        "Programming Language :: Python :: 3",
+    ],
+    keywords="bloom filter cascade multi level mlbf",
+    url="http://github.com/mozmark/filter-cascade",
+    author="Mark Goodwin",
+    author_email="[email protected]",
+    license="Mozilla Public License 2.0 (MPL 2.0)",
+    packages=["filtercascade"],
+    install_requires=["bitarray>=0.9.2", "mmh3>=2.5.1",],
+    include_package_data=True,
+    zip_safe=False,
+)