Merge pull request #2 from dClimate/evan-dev

Evan dev

Author: eschechter

.gitignore

@@ -0,0 +1,153 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/

README.md

@@ -0,0 +1,14 @@
+# python-iamap
+
+## python implementation of a HAMT, adapted from rvagg's JavaScript version
+
+See https://github.com/rvagg/iamap#readme for details on a JS implementation of this code. This python version is adapted from rvagg's code; there are one-to-one mappings between functions/classes in this repo and those in the JS one. As a result, the JS code can serve as a canonical guide to implementation and functionality.
+
+See https://ipld.io/specs/advanced-data-layouts/hamt/spec/ for information on the concept of a HAMT and how it fits into the IPLD/IPFS ecosystem. 
+
+
+## Motivation
+
+dClimate uses HAMTs as key/value stores that can be distributed across multiple nodes and used without the whole data structure being loaded into memory. This is extremely useful in the context of [zarrs](https://zarr.readthedocs.io/en/stable/), where metadata mapping coordinates to chunks containing the actual data can stretch into the 10s or even 100s of MBs. Because IPFS imposes a limit on the sizes of blocks that can be transferred from peer to peer, it is not feasible to store all this metadata in a single IPFS object. Instead, a HAMT can be used to provide efficient lookups in a data structure distributed across many IPFS objects, with only the parts of the HAMT needed for the lookup ever being accessed. 
+
+See [ipldstore](https://github.com/dClimate/ipldstore/tree/hamt) for an example of this HAMT implementation in action.

examples/ipfs_store.py

@@ -0,0 +1,31 @@
+import requests
+import dag_cbor
+import cbor2
+from multiformats import CID
+
+
+class HamtIPFSStore:
+    def save(self, obj):
+        obj = dag_cbor.encode(obj)
+        res = requests.post(
+            "http://localhost:5001/api/v0/dag/put",
+            params={"store-codec": "dag-cbor", "input-codec": "dag-cbor", "pin": False},
+            files={"dummy": obj},
+        )
+        res.raise_for_status()
+        return CID.decode(res.json()["Cid"]["/"])
+
+    def load(self, id):
+        if isinstance(id, cbor2.CBORTag):
+            id = CID.decode(id.value[1:])
+        res = requests.post(
+            "http://localhost:5001/api/v0/block/get", params={"arg": str(id)}
+        )
+        res.raise_for_status()
+        return dag_cbor.decode(res.content)
+
+    def is_equal(self, id1: CID, id2: CID):
+        return str(id1) == str(id2)
+
+    def is_link(self, obj: CID):
+        return isinstance(obj, CID) and obj.codec.name == "dag-cbor"

py_hamt/__init__.py

@@ -0,0 +1,110 @@
+import math
+
+
+def extract_bits(hash_obj: bytes, depth: int, nbits: int) -> int:
+    """Extract `nbits` bits from `hash_obj`, beginning at position `depth * nbits`,
+    and convert them into an unsigned integer value.
+
+    Args:
+        hash_obj (bytes): binary hash to extract bit sequence from
+        depth (int): depth of the node containing the hash
+        nbits (int): bit width of hash
+
+    Returns:
+        int: An unsigned integer version of the bit sequence
+    """
+    start = depth * nbits
+    start_offset = start % 8
+
+    byte_count = math.ceil((start_offset + nbits) / 8)
+    byte_start = start >> 3
+    end_offset = byte_count * 8 - nbits - start_offset
+
+    result = 0
+
+    for i in range(byte_count):
+        local = hash_obj[byte_start + i]
+        shift = 0
+        local_bit_length = 8
+
+        if i == 0:
+            local_bit_length -= start_offset
+
+        if i == byte_count - 1:
+            local_bit_length -= start_offset
+            shift = end_offset
+            local >>= shift
+
+        if local_bit_length < 8:
+            m = (1 << local_bit_length) - 1
+            local &= m
+
+        if shift < 8:
+            result = result << (8 - shift)
+        result |= local
+
+    return result
+
+
+def set_bit(bitmap: bytes, position: int, to_set: bool) -> bytes:
+    """set the `position` bit in the given `bitmap` to be `to_set` (truthy=1, falsey=0)
+
+    Args:
+        bitmap (bytes): bitmap to modify
+        position (int): location in the bitmap to modify
+        to_set (bool): whether to set true or false
+
+    Returns:
+        bytes: Modified bitmap
+    """
+    has = bitmap_has(bitmap, position)
+    byte = math.floor(position / 8)
+    offset = position % 8
+    # if we assume that `bitmap` is already the opposite of `set`, we could skip this check
+    if (to_set and not has) or (not to_set and has):
+        new_bit_map = bytearray(bitmap)
+        b = bitmap[byte]
+        if to_set:
+            b |= 1 << offset
+        else:
+            b ^= 1 << offset
+
+        # since bytes are immutable, we need to change bytes to bytearrays
+        new_bit_map[byte] = b
+        return bytes(new_bit_map)
+    return bitmap
+
+
+def bitmap_has(
+    bitmap: bytes,
+    position: int,
+) -> bool:
+    """check whether `bitmap` has a `1` at the given `position` bit.
+
+    Args:
+        bitmap (bytes): bytes to check
+        position (int): Position of bit to read
+
+    Returns:
+        bool: whether the `bitmap` has a 1 value at the `position` bit
+    """
+    byte = math.floor(position / 8)
+    offset = position % 8
+    return ((bitmap[byte] >> offset) & 1) == 1
+
+
+def rank(bitmap: bytes, position: int) -> int:
+    """count how many `1` bits are in `bitmap` up until `position`
+    tells us where in the compacted element array an element should live
+    Args:
+        bitmap (bytes): bitmap to count truthy bits on
+        position (int): where to stop counting
+
+    Returns:
+        int: how many bits are `1` in `bitmap`
+    """
+    t = 0
+    for i in range(position):
+        if bitmap_has(bitmap, i):
+            t += 1
+    return t

py_hamt/bit_utils.py

@@ -0,0 +1,11 @@
+# pyproject.toml
+[build-system]
+requires = [
+    "setuptools>=45",
+    "wheel",
+    "setuptools_scm>=6.2"
+]
+build-backend = "setuptools.build_meta"
+[tool.setuptools_scm]
+version_scheme = "post-release"
+local_scheme = "no-local-version"

py_hamt/hamt.py

@@ -0,0 +1,33 @@
+
+
+# https://packaging.python.org/en/latest/tutorials/packaging-projects/
+# https://setuptools.pypa.io/en/latest/userguide/declarative_config.html
+# https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#setup-cfg
+
+
+[metadata]
+name = py-hamt
+url = https://github.com/dClimate/py-iamap
+
+[options]
+packages = find:
+python_requires = >=3.8
+
+[options.extras_require]
+dev = 
+    flake8
+    black
+
+[options.package_data]
+* = py.typed, *.json
+
+[options.packages.find]
+exclude =
+    test
+    examples
+
+[flake8]
+max-line-length = 119
+
+[tool.black]
+line-length = 119

pyproject.toml

@@ -0,0 +1,95 @@
+from py_hamt import bit_utils
+
+# mask_fun tests
+assert bit_utils.extract_bits(bytes([0b11111111]), 0, 5) == 0b11111
+assert bit_utils.extract_bits(bytes([0b10101010]), 0, 5) == 0b10101
+assert bit_utils.extract_bits(bytes([0b10000000]), 0, 5) == 0b10000
+assert bit_utils.extract_bits(bytes([0b00010000]), 0, 5) == 0b00010
+
+# bitmap_has tests
+assert not bit_utils.bitmap_has(bytes([0b0]), 0)
+assert not bit_utils.bitmap_has(bytes([0b0]), 1)
+assert bit_utils.bitmap_has(bytes([0b1]), 0)
+assert not bit_utils.bitmap_has(bytes([0b1]), 1)
+assert not bit_utils.bitmap_has(bytes([0b101010]), 2)
+assert bit_utils.bitmap_has(bytes([0b101010]), 3)
+assert not bit_utils.bitmap_has(bytes([0b101010]), 4)
+assert bit_utils.bitmap_has(bytes([0b101010]), 5)
+assert bit_utils.bitmap_has(bytes([0b100000]), 5)
+assert bit_utils.bitmap_has(bytes([0b0100000]), 5)
+assert bit_utils.bitmap_has(bytes([0b00100000]), 5)
+print("bitmap_has tests passed")
+
+# index tests
+assert bit_utils.rank(bytes([0b111111]), 0) == 0
+assert bit_utils.rank(bytes([0b111111]), 1) == 1
+assert bit_utils.rank(bytes([0b111111]), 2) == 2
+assert bit_utils.rank(bytes([0b111111]), 4) == 4
+assert bit_utils.rank(bytes([0b111100]), 2) == 0
+assert bit_utils.rank(bytes([0b111101]), 4) == 3
+assert bit_utils.rank(bytes([0b111001]), 4) == 2
+assert bit_utils.rank(bytes([0b111000]), 4) == 1
+assert bit_utils.rank(bytes([0b110000]), 4) == 0
+# new node, no bitmask_fun, insertion at the start
+assert bit_utils.rank(bytes([0b000000]), 0) == 0
+assert bit_utils.rank(bytes([0b000000]), 1) == 0
+assert bit_utils.rank(bytes([0b000000]), 2) == 0
+assert bit_utils.rank(bytes([0b000000]), 3) == 0
+print("index tests passed")
+
+# set_bit tests
+assert bit_utils.set_bit(bytes([0b0]), 0, 1) == bytes([0b00000001])
+assert bit_utils.set_bit(bytes([0b0]), 0, 1) == bytes(([0b00000001]))
+assert bit_utils.set_bit(bytes([0b0]), 1, 1) == bytes(([0b00000010]))
+assert bit_utils.set_bit(bytes([0b0]), 7, 1) == bytes(([0b10000000]))
+assert bit_utils.set_bit(bytes([0b11111111]), 0, 1) == bytes(([0b11111111]))
+assert bit_utils.set_bit(bytes([0b11111111]), 7, 1) == bytes(([0b11111111]))
+assert bit_utils.set_bit(bytes([0b01010101]), 1, 1) == bytes(([0b01010111]))
+assert bit_utils.set_bit(bytes([0b01010101]), 7, 1) == bytes(([0b11010101]))
+assert bit_utils.set_bit(bytes([0b11111111]), 0, 0) == bytes(([0b11111110]))
+assert bit_utils.set_bit(bytes([0b11111111]), 1, 0) == bytes(([0b11111101]))
+assert bit_utils.set_bit(bytes([0b11111111]), 7, 0) == bytes(([0b01111111]))
+assert bit_utils.set_bit(bytes([0b0, 0b11111111]), 8 + 0, 1) == bytes(
+    ([0b0, 0b11111111])
+)
+assert bit_utils.set_bit(bytes([0b0, 0b11111111]), 8 + 7, 1) == bytes(
+    ([0b0, 0b11111111])
+)
+assert bit_utils.set_bit(bytes([0b0, 0b01010101]), 8 + 1, 1) == bytes(
+    ([0b0, 0b01010111])
+)
+assert bit_utils.set_bit(bytes([0b0, 0b01010101]), 8 + 7, 1) == bytes(
+    ([0b0, 0b11010101])
+)
+assert bit_utils.set_bit(bytes([0b0, 0b11111111]), 8 + 0, 0) == bytes(
+    ([0b0, 0b11111110])
+)
+assert bit_utils.set_bit(bytes([0b0, 0b11111111]), 8 + 1, 0) == bytes(
+    ([0b0, 0b11111101])
+)
+assert bit_utils.set_bit(bytes([0b0, 0b11111111]), 8 + 7, 0) == bytes(
+    ([0b0, 0b01111111])
+)
+assert bit_utils.set_bit(bytes([0b0]), 0, 0) == bytes(([0b00000000]))
+assert bit_utils.set_bit(bytes([0b0]), 7, 0) == bytes(([0b00000000]))
+assert bit_utils.set_bit(bytes([0b01010101]), 0, 0) == bytes(([0b01010100]))
+assert bit_utils.set_bit(bytes([0b01010101]), 6, 0) == bytes(([0b00010101]))
+assert bit_utils.set_bit(
+    bytes([0b11000010, 0b11010010, 0b01001010, 0b0000001]), 0, 0
+) == bytes(([0b11000010, 0b11010010, 0b01001010, 0b0000001]))
+assert bit_utils.set_bit(
+    bytes([0b11000010, 0b11010010, 0b01001010, 0b0000001]), 0, 1
+) == bytes(([0b11000011, 0b11010010, 0b01001010, 0b0000001]))
+assert bit_utils.set_bit(
+    bytes([0b11000010, 0b11010010, 0b01001010, 0b0000001]), 12, 0
+) == bytes(([0b11000010, 0b11000010, 0b01001010, 0b0000001]))
+assert bit_utils.set_bit(
+    bytes([0b11000010, 0b11010010, 0b01001010, 0b0000001]), 12, 1
+) == bytes(([0b11000010, 0b11010010, 0b01001010, 0b0000001]))
+assert bit_utils.set_bit(
+    bytes([0b11000010, 0b11010010, 0b01001010, 0b0000001]), 24, 0
+) == bytes(([0b11000010, 0b11010010, 0b01001010, 0b0000000]))
+assert bit_utils.set_bit(
+    bytes([0b11000010, 0b11010010, 0b01001010, 0b0000001]), 24, 1
+) == bytes(([0b11000010, 0b11010010, 0b01001010, 0b0000001]))
+print("all tests passed")