diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 21e74bd01..3b6946592 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -13,7 +13,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.9' cache: 'pip' - run: python -m pip install -r requirements.txt - uses: ./.github/actions/coverage @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' cache: 'pip' - uses: actions/cache/restore@v4 id: restore_cache diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 64830abfd..8dcb9b2e7 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup Python 3.8 + - name: Setup Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install mypy run: pip install mypy - name: Run mypy diff --git a/.github/workflows/pr-comment-validate.yml b/.github/workflows/pr-comment-validate.yml index 07d966fb9..e59153353 100644 --- a/.github/workflows/pr-comment-validate.yml +++ b/.github/workflows/pr-comment-validate.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, windows-latest] # macos-latest not tested due to crashing. - version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -72,7 +72,7 @@ jobs: set_failure_status: if: ${{failure() && github.event.issue.pull_request && startsWith(github.event.comment.body, 'validate') }} - needs: [set_pending_status, pr_commented, validate_attestation_tutorial, validate_identity_tutorial, validate_documentation] + needs: [set_pending_status, pr_commented, validate_identity_tutorial, validate_documentation] runs-on: ubuntu-latest steps: - name: Set commit failed status @@ -89,7 +89,7 @@ jobs: set_success_status: if: ${{!failure() && github.event.issue.pull_request && startsWith(github.event.comment.body, 'validate') }} - needs: [set_pending_status, pr_commented, validate_attestation_tutorial, validate_identity_tutorial, validate_documentation] + needs: [set_pending_status, pr_commented, validate_identity_tutorial, validate_documentation] runs-on: ubuntu-latest steps: - name: Set commit success status @@ -104,30 +104,6 @@ jobs: sha: ${{ env.actual_pull_head }} target_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - validate_attestation_tutorial: - name: Validate attestation tutorial - if: ${{github.event.issue.pull_request && startsWith(github.event.comment.body, 'validate') }} - needs: set_pending_status - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: refs/pull/${{ github.event.issue.number }}/head - - name: Setup Python 3.8 - uses: actions/setup-python@v5 - with: - python-version: 3.8 - - name: Install dependencies - run: | - pip install --upgrade setuptools pip - pip install -r requirements.txt - - name: Run attestation tests - run: | - export PYTHONPATH="$(pwd)/" - cd doc/deprecated/attestation_tutorial_integration - python attestation_tutorial_attest.py - python attestation_tutorial_verify.py - validate_identity_tutorial: name: Validate identity tutorial if: ${{github.event.issue.pull_request && startsWith(github.event.comment.body, 'validate') }} @@ -137,10 +113,10 @@ jobs: - uses: actions/checkout@v4 with: ref: refs/pull/${{ github.event.issue.number }}/head - - name: Setup Python 3.8 + - name: Setup Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | pip install --upgrade setuptools pip @@ -161,10 +137,10 @@ jobs: - uses: actions/checkout@v4 with: ref: refs/pull/${{ github.event.issue.number }}/head - - name: Setup Python 3.8 + - name: Setup Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install dependencies run: | pip install --upgrade setuptools pip diff --git a/.github/workflows/python-publish-taskmanager.yml b/.github/workflows/python-publish-taskmanager.yml deleted file mode 100644 index d6f081920..000000000 --- a/.github/workflows/python-publish-taskmanager.yml +++ /dev/null @@ -1,33 +0,0 @@ -# This workflow will upload a standalone TaskManager Python Package using Twine when manually triggered -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: Upload TaskManager Python Package - -on: - workflow_dispatch: - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD_TASKMANAGER }} - run: | - python create_setup_taskmanager.py - pushd ipv8_taskmanager - python setup.py sdist bdist_wheel - twine upload dist/* - popd diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 92bd94975..38901f6b1 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -5,10 +5,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Setup Python 3.8 + - name: Setup Python 3.9 uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.9 - name: Install ruff run: pip install ruff - name: Get changed Python files diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml index af3423970..6cc169ce3 100644 --- a/.github/workflows/unittests.yml +++ b/.github/workflows/unittests.yml @@ -7,7 +7,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.7' + python-version: '3.9' cache: 'pip' - run: python -m pip install -r requirements.txt - name: Run unit tests @@ -18,7 +18,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' cache: 'pip' - uses: actions/cache/restore@v4 id: restore_cache diff --git a/.ruff.toml b/.ruff.toml index 7200c4c18..a6e15d897 100644 --- a/.ruff.toml +++ b/.ruff.toml @@ -89,7 +89,7 @@ line-length = 120 # Allow unused variables when underscore-prefixed. lint.dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" -target-version = "py37" +target-version = "py39" [lint.pylint] max-args = 6 diff --git a/README.md b/README.md index d03e92eb8..441bf29d1 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,6 @@ Running tests can be done by running: python3 run_all_tests.py ``` -*Running the test suite on Python 3.7 requires the installation of `asynctest` (`python3 -m pip install asynctest`).* - Running code coverage requires the `coverage` package (`python3 -m pip install coverage`). A coverage report can be generated by running: diff --git a/create_setup_taskmanager.py b/create_setup_taskmanager.py deleted file mode 100644 index f4dd3bd92..000000000 --- a/create_setup_taskmanager.py +++ /dev/null @@ -1,216 +0,0 @@ -""" -Extract and rewrite ``ipv8.taskmanager`` to be a standalone package (``ipv8_taskmanager``). - -Install your newly created package using `cd ipv8_taskmanager && pip install .` -""" -from __future__ import annotations - -import ast -import importlib -import logging -import os -import typing - -import ipv8.taskmanager - -MissingImport = typing.NamedTuple("MissingImport", ["node", "stmt", "alias"]) -TASKMNGR_FILE = ipv8.taskmanager.__file__ -TASKMNGR_FOLDER = os.path.dirname(TASKMNGR_FILE) -TARGET_TMP_DIR = "ipv8_taskmanager" -TARGET_FOLDER = os.path.join(TARGET_TMP_DIR, "ipv8_taskmanager") -TARGET_FILE = os.path.join(TARGET_FOLDER, "__init__.py") -LINESEP = "\n" - - -def prepare_folder() -> None: - """ - Make sure the target folder does not exist. - We don't want a user to accidentally overwrite his files. - - :returns: None - """ - try: - os.makedirs(TARGET_FOLDER, mode=0o777, exist_ok=False) - except FileExistsError as e: - logging.exception("The ipv8_taskmanager folder already exists. " - "Please check that you are not overwriting something important and " - "delete the folder manually.") - raise e # noqa: TRY201 - - -def find_missing_imports(tree: ast.AST) -> list[MissingImport]: - """ - Try to find any imports that the tree requires. - - We can inject ``from x import y`` imports straight into the source file. - We error out on ``import x`` imports. - "injectable-but-missing" imports are returned as ``MissingImport`` tuples. - - :return: the list of missing imports - """ - missing = [] - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - stmt = f"import {alias.name}" - try: - exec(stmt, {}, {}) # noqa: S102 - except ModuleNotFoundError as e: - logging.exception("Unimportable modules are not allowed!") - raise e # noqa: TRY201 - elif isinstance(node, ast.ImportFrom): - for alias in node.names: - stmt = f"from {node.module} import {alias.name}" - try: - exec(stmt, {}, {}) # noqa: S102 - except ModuleNotFoundError: - missing.append(MissingImport(node=node, stmt=stmt, alias=alias.name)) - return missing - - -def copy_node_as_src(node: ast.AST, lines: dict[int, str]) -> str: - """ - Get the source code an AST node is defined on. - - :return: the source code of the given node - """ - output = "" - for l in range(node.lineno, node.end_lineno + 1): - output += lines[l] + LINESEP - return output - - -def fetch(missing: MissingImport) -> tuple[str, typing.Set[str], typing.Set[str]]: - """ - Get the code required to satisfy a missing import. - - For example, if you ``from a import b``. This will return all imports, module level definitions and the - function ``b``. Suppose you have the following ``a.py``, you would get the lines marked with ``YES``: - - .. code-block :: Python - - import math # YES - SOME_GLOBAL = 1 # YES - def b(): # YES - return 1 # YES - def c(): # NO - return 2 # NO - - :param missing: the import to fetch the required code for - :return: the code required for the missing import - """ - missing_module = importlib.import_module("." * missing.node.level + missing.node.module, "ipv8") - - with open(missing_module.__file__) as source_file: - source = source_file.read() - tree = ast.parse(source, filename=missing_module.__file__) - lines = {i + 1: line for i, line in enumerate(source.split(LINESEP))} - - output = "" - imports = set() - toplevel_vars = set() - - for node in tree.body: - if isinstance(node, (ast.Import, ast.ImportFrom)): - imports.add(copy_node_as_src(node, lines)) - elif isinstance(node, ast.Assign): - toplevel_vars.add(copy_node_as_src(node, lines)) - elif isinstance(node, ast.FunctionDef) and node.name == missing.alias: - output += LINESEP + LINESEP + copy_node_as_src(node, lines) - - return output, imports, toplevel_vars - - -def read_tskmngr_source() -> tuple[ast.AST, str, dict[int, str]]: - """ - Parse the ``ipv8.taskmanager`` code and return an AST node and the source code as both str and list-of-lines form. - - :return: the ast node, the full source code, the source code as a list of lines - """ - with open(TASKMNGR_FILE) as source_file: - source = source_file.read() - tree = ast.parse(source, filename=TASKMNGR_FILE) - - lines = {i + 1: line for i, line in enumerate(source.split(LINESEP))} - - return tree, source, lines - - -def extract_taskmanager() -> None: - """ - Rewrite the ``ipv8.taskmanager`` to be standalone. - Put the result in ``ipv8_taskmanager/ipv8_taskmanager``. - - :returns: None - """ - prepare_folder() - - tree, source, lines = read_tskmngr_source() - missing_imports = find_missing_imports(tree) - - for replaced in {missing.node.lineno for missing in missing_imports}: - lines.pop(replaced) - - new_toplevel_imports = set() - new_toplevel_vars = set() - for missing in missing_imports: - line_num = missing.node.lineno - source = lines.get(line_num, "") - missing_output, nimports, ntoplevel_vars = fetch(missing) - new_toplevel_imports |= nimports - new_toplevel_vars |= ntoplevel_vars - source += missing_output - lines[line_num] = source - - # Add imports - lines[0] = "".join(new_toplevel_imports) + LINESEP + "".join(new_toplevel_vars) - - # Turn into source code - raw_source = "".join(lines[l] + LINESEP for l in sorted(lines)) - - # Write to file - with open(TARGET_FILE, "w") as f: - f.write(raw_source) - - -def make_setup() -> None: - """ - Create ``ipv8_taskmanager/setup.py``. - - :returns: None - """ - with open(os.path.join(TARGET_TMP_DIR, "setup.py"), "w") as f: - f.write("""from setuptools import find_packages, setup -setup( - name='ipv8_taskmanager', - author='Tribler', - description='The IPv8 TaskManager', - long_description=('This module provides a set of tools to maintain a list of asyncio Tasks that are to be ' - 'executed during the lifetime of an arbitrary object, usually getting killed with it. This ' - 'module is extracted from the IPv8 main project.'), - long_description_content_type='text/markdown', - version='1.0.0', - url='https://github.com/Tribler/py-ipv8', - package_data={'': ['*.*']}, - packages=find_packages(), - py_modules=[], - install_requires=[], - extras_require={}, - tests_require=[], - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Topic :: Software Development :: Libraries :: Python Modules" - ] -) -""") - - -if __name__ == "__main__": - extract_taskmanager() - make_setup() diff --git a/doc/basics/identity_tutorial_integration/main.py b/doc/basics/identity_tutorial_integration/main.py index 9db94ab14..ebf67ba22 100644 --- a/doc/basics/identity_tutorial_integration/main.py +++ b/doc/basics/identity_tutorial_integration/main.py @@ -1,6 +1,5 @@ -from asyncio import run +from asyncio import run, sleep from base64 import b64encode -from time import sleep from ipv8.configuration import get_default_configuration from ipv8.REST.rest_manager import RESTManager @@ -30,7 +29,7 @@ async def start_community() -> None: await rest_manager.start(14410 + peer_id) keep_trying = False except OSError: - sleep(1.0) # noqa: ASYNC101 + await sleep(1.0) # Print the peer for reference print("Starting peer", b64encode(ipv8.keys["anonymous id"].mid)) diff --git a/doc/deprecated/attestation_prototype.rst b/doc/deprecated/attestation_prototype.rst deleted file mode 100644 index ab19e0a8c..000000000 --- a/doc/deprecated/attestation_prototype.rst +++ /dev/null @@ -1,378 +0,0 @@ - -Attestation Prototype Documentation -=================================== - -This document describes the original design of the IPv8-based Android application and credential authority. - -Passive Updates in the IPv8 Android Application: ------------------------------------------------- - -Example behaviour of *passive* and *regular* updates in an **IPv8 Android Application** instance, regarding fellow peer lists and their associated attributes. It should be noted that the **Attestation Server** is nothing more than a *well-known peer* itself, accessed through an HTTP URL (\ ``127.0.0.1:8086/attestation``\ ): - - -.. image:: ./resources/attestation_req.png - :target: ./resources/attestation_req.png - :alt: Alt Text - - -**Every second**\ , the application will do the following: - - -#. - Forward a (\ **GET**\ : *peers*\ ) request to a certain peer, with the aim of updating the local list of known peers. - -#. - When the response for the previous request arrives, containing unique identifiers (IDs) for the fellow peers, update the local list of known peers. Simultaneously, forward a (\ **GET**\ : *attributes*\ ) for each peer ID, in order to obtain a list of the transaction names and hashes as found in the latest 200 blocks associated to the peer. The information in these responses is stored locally as well. - -#. - Forward a (\ **GET**\ : *attributes*\ ) request to and for the peer contacted in **Step 1**. Similarly to **Step 2**\ , the response should contain the transaction names and hashes as found in the latest 200 blocks associated to the peer. This information is stored locally. - -Flow of Communication in Active Attestation Requests: ------------------------------------------------------ - -Example RESTful based communication flow for serving **Attestation Requests**\ , as currently implemented in the demo application: - - -.. image:: ./resources/output_SQjlvW.gif - :target: ./resources/output_SQjlvW.gif - :alt: Alt Text - - - -#. - The **Client**\ 's user interacts with their device in order to **obtain attestation** for a particular **attribute**. - -#. - The **Client** attempts to update the list of known peers by forwarding a (\ **GET**\ : *peers*\ ) request to a well-known fellow peer (the **Attestation Server**\ ). - -#. - The **Client** demands **attestation** for the **attribute**\ , by forwarding a (\ **POST**\ : *request*\ ) request for the particular **attribute** to every known peer, as identified by their unique ID. In this particular demo implementation, only the well-known peer (the **Attestation Server**\ ) is forwarded the request. - -#. - A **Fellow Peer** will await for at least one other fellow peer (the **Client**\ ) to be present in the network, before accepting and solving attestation requests. It will do so by repeatedly forwarding (\ **GET**\ : *peers*\ ) requests to the well-known peer, until this returns a non-empty list of peers. - -#. - The **Fellow Peer** will probe for **outstanding** (i.e. as of yet unresolved) **attestation requests**. To obtain such a list, it forwards a (\ **GET**\ : *outstanding*\ ) request to the well-known peer. The request will return a list of tuples of the form ``(, )``\ , each representing an **outstanding request**. - -#. - The **Fellow Peer** chooses to solve the request, and **attest** the requested **attribute**. This process is done locally, in the peer itself. The result of the attestation is encoded in *Base64* format, and packed in a (\ **POST**\ : *attest*\ ) request, which is forwarded to the well-known peer (the **Attestation Server**\ ). - -#. - The well-known peer (the **Attestation Server**\ ), returns the **attestation** to the **Client**\ , as a response to their original (\ **POST**\ : *request*\ ) **attestation request**. - -#. - The **Client** receives the **attestation**\ , and displays the **successful attestation** status to the device user. - -Peers and their REST APIs: --------------------------- - -It should be mentioned that the protocol, which abides to the REST paradigms, used towards facilitating communication between peers, in this particular implementation, is the **Hypertext Transfer Protocol (HTTP)**. - -The diagram below describes the **REST API** of an **IPv8** object (in this current implementation version). These are requests which can be handled by the **IPv8** objects: - - -.. image:: ./resources/peer_rest_api.png - :target: ./resources/peer_rest_api.png - :alt: Alt Text - - -Below, a detailed explanation of the REST Requests is presented: - - -* - **GET Methods** sent to the **AttestationEndpoint**\ : - - - * - ``type = "outstanding"`` : Retrieve a list of tuples of the form ``(, )``\ , each representing an, as of yet, unresolved attestation request (from a peer identified by ```` for the attribute ````\ ). - - * - ``type = "verification_output"`` : Retrieve a dictionary (equivalently, a map) of the form: `` -> (, )``. The dictionary should contain a record of all the previous verification processes' results. - - * - ``type = "peers"`` : Retrieve a list of Base64 encoded peer IDs, which uniquely identify the peers in the **first Identity Overlay**. - - * - ``type = "attributes"`` : Return a list of tuples of the form ``(, )``\ , from the the latest 200 blocks associated with a **particular peer**. - - *Optional parameters* : - - - * ``mid = `` : Identifies the **particular peer** (using its unique ID) whose attributes are being requested. If this parameter is missing, the **peer** shall default to the target endpoint's peer. - -* - **POST Methods** sent to the **AttestationEndpoint**\ : - - - * - ``type = "request"`` : Request the attestation of a particular attribute from a particular peer. - - *Required additional parameters* : - - - * - ``mid = ``\ : Identifies the peer (by its unique ID) from which the attestation is requested. - - * - ``attribute_name = ``\ : The name of the attribute, for which attestation is requested. - - * - ``type = "attest"`` : Attest a particular attribute, for a particular peer. Additionally, return an attested value for the aforementioned attribute. - - *Required additional parameters* : - - - * - ``mid = ``\ : Identifies the peer (by its unique ID) from which the attestation is requested. - - * - ``attribute_name = ``\ : The name of the attribute, for which attestation is requested. - - * - ``attribute_value = ``\ : Base64 encoded attested value for the attribute. - - * - ``type = "verify"`` : Request the verification of a particular attestation and set of values associated to a particular attribute, from another particular peer. - - *Required additional parameters* : - - - * - ``mid = ``\ : Identifies the peer (by its unique ID) from which the attestation is requested. - - * - ``attribute_hash = ``\ : A Base64 encoded hash of the attribute's particular attestation, subject to verification. - - * - `attribute_values = `: A variable length list of Base64 encoded attribute values, for which verification is required. `` is a comma separated string of values, that is: `` = ",,...,"``\ , where ``N`` may be arbitrarily large. - -* - **GET Methods** sent to the **NetworkEndpoint**\ : - - - * **Any GET request**\ : Retrieve a dictionary (equivalently, a map) holding information on each verified peer in the network. The dictionary's structure will be the following: `` -> {"id": , "port": , "public_key": , "services": }``. ```` is a list which identifies the known services supported by the peer. The dictionary itself is returned within a dictionary: ``{"peers": }``. - -The diagram below describes the **REST Requests** implemented in the **Android Application** (in this current implementation version). These requests are forwarded to, and handled by the **IPv8** object's **REST APIs**\ : - - -.. image:: ./resources/android_rest_api.png - :target: ./resources/android_rest_api.png - :alt: Alt Text - - -All requests must abide to the specifications detailed above (in this section). - -Attestation Process: Detailed Explanation ------------------------------------------ - -Below is a detailed description of the general flow for handling **incoming attestation requests**. It should be noted that this description is general, and does not necessarily refer to this particular implementation version (i.e. of the demo). Moreover, the reader shoould also note that **attestation** is requested from a particular peer (see documentation above for (\ **POST**\ ): *attest* @ **AttestationEndpoint**\ ), however, the actual HTTP request is forwarded to another peer, potentially different from the **attester** itself. Hence, there shall be two cases for the **attestation request**\ : - - -#. - When a peer receives the HTTP request, packs it, and forwards it to the peer from which attestation is requested (i.e. the **attester** peer). - -#. - When the peer from which attestation is requested (i.e. the **attester** peer) receives the attestation request itself, as obtained from the 1st case. - -The two cases shall be further described in the sections to follow. To better understand what each of them do, however, we must introduce two callback methods of the ``AttestationCommunity`` class, located in the **community.py** module. The two callbacks are stored in the ``self.attestation_request_callbacks`` field, which is an array that should contain 2 elements, intuitively, each being a callback. The methods which set the callbacks are described in the following (both are located in the ``AttestationCommunity`` class): - - -* - `set_attestation_request_callback(f)`: this is the method which stores the function `f` in `self.attestation_request_callbacks[0]` field. `f` is the method actually handles the **attestation**. This method is called in `AttestationCommunity`'s `on_request_attestation` method when some peer requests the attestation from us (i.e. we are the **attester**). `f` must return a string if the attestation is made, this represents the attestation value. If `None` is returned, however, the attestation is not made. As input, `f` receives the following: - - - * - ``: Peer``\ : A ``Peer`` object representing the peer which forwarded this request to us. - - * - ``: string``\ : A ``string`` object, representing the name of the attribute requiring attestation. - -* - `set_attestation_request_complete_callback(f)`: this is the method which stores the function `f` in `self.attestation_request_callbacks[1]` field. `f` is called back when an attestation has been completed.`f` needn't return anything. It should accept the following input parameters: - - - * - ``: Peer``\ : A ``Peer`` object representing the peer which forwarded this request to us. - - * - ``: string``\ : A ``string`` object, representing the name of the attribute requiring attestation. - - * - ``: string``\ : A ``string`` object, representing the hash of the attestation blob. - - * - `` = None: string``\ : A ``Peer`` object representing the signer of the attestation (i.e. usually us). - -Below is a description of the two scenarios of the attestation process presented above. **Make sure to also read this section's last subsection. It contains details on how the attestation process works for this demo implementation**. - -Receiving the HTTP Attestation Request -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the **community.py** module, and the ``AttestationCommunity`` class, the method ``request_attestation`` is called by the ``render_POST`` method in the ``AttestationEndpoint`` class, upon receiving a (\ **POST**\ : *request*\ ) HTTP request. Attestation might be required from the peer receiving this request, or from another peer. Regardless, the ``request_attestation`` method will do the following: - - -#. - It will add the request to its own *request cache*. - -#. - It will create some **metadata** from the request, that is: the *attribute's name* to be attested, and the request's *public key*. - -#. - It will create 3 payloads: an **authentication payload** (communication security between this peer and the **attester**\ ), the actual **attestation request payload** (created from the metadata), and the **time distribution payload** (created from the global time). The payloads are packed together in a packet which is forwarded to the attester. - -It should be noted that this method should normally be called once, when a raw (\ **POST**\ : *request*\ ) HTTP request is received. This method will send a packet (with the contents as described above) directly to the **attester** peer, which might, in fact, be this peer, or another different peer. - -Receiving the Attestation Request in the Attester Peer -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In the **community.py** module and the ``AttestationCommunity`` class, the method ``on_request_attestation`` is called when attestation is requested from us. That is, when an attestation packet, as created by the ``request_attestation`` method in the ``AttestationCommunity`` is received. The ``on_request_attestation`` method will do the following: - - -#. - Unpack the packet, and get the 3 payloads: **attestation payload**\ , **request payload**\ , and **time distribution payload**. The source of the request is seen as the peer which last forwarded the request. - -#. - The attestation payload's metadata is retrieved and is used towards the attestation process. Attestation is performed by calling the ``self.attestation_request_callbacks[0]`` with the *source peer* and *attribute name* as parameters. If a value is returned, attestation has been performed, otherwise attestation has not been performed. - -#. - A blob is created from the attestation value. The blob is first used as a parameter in the ``self.attestation_request_callbacks[1]``\ , together with the *source peer*\ , and the *attribute name*. After returning from the aforementioned call, the attestation blob is sent to the *source peer* (i.e. the peer from which we received this request). - -Side Note: Detailed Attestation Flow in the Demo -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This case is a bit curious, since the peer forwarding the (\ **POST**\ : *attest*\ ) HTTP request (i.e. the peer running in the **main.py** script, in the method ``make_attribute``\ ), is indeed solving the **attestation request**\ , however, it is not actually holding it. In the demo, the peer holding the request is the well-known peer behind ``localhost:8086/attestation`` URL. The agent running in the **main.py** script is made aware of the request by having previously forwarded a (\ **GET**\ : *outstanding*\ ) HTTP request to the well-known peer. It will serve the attestation request by forwarding a (\ **POST**\ : *attest*\ ) request back to the ``localhost:8086/attestation`` (well-known) peer. - -As previously mentioned, the well-known peer behind ``localhost:8086/attestation`` is actually holding the **attestation requests**. For each request forwarded to this peer, a ``Deferred`` object is created, which is attached to a ``yield``. This is done in the ``on_request_attestation`` method (of the ``AttestationEndpoint`` class), which is called as ``self.attestation_request_callbacks[0]`` in ``on_request_attestation`` method (of the ``AttestationCommunity`` class). The ``yield`` will suspend the thread, which will now wait for the ``Deferred`` object to yield something. For this object to yield something, someone has to call the ``Deferred.callback()`` method on it. This is done when the agent behind **main.py** forwards a (\ **POST**\ : *attest*\ ) HTTP request to the well-known peer. In the ``render_POST`` method of ``AttestationEndpoint`` class, for this ``type`` of request, the deferred object is retrieved, and called back with an attestation value (obtained from the (\ **POST**\ : *attest*\ ) request) as the ```` parameter. This allows the ``Deferred`` object to yield this value, and, in turn, allows the ``on_request_attestation`` (of ``AttestationCommunity``\ ) method to continue executing, which was halted by its call to ``self.attestation_request_callbacks[0]``\ , i.e. ``on_request_attestation`` (of ``AttestationEndpoint``\ ). The ``on_request_attestation`` (of ``AttestationCommunity``\ ) eventually calls ``self.attestation_request_callbacks[1]``\ , i.e. ``on_attestation_complete``\ , and finally sends back the attestation to the source peer. - -Annex: General Notes on Demo Classes ------------------------------------- - -Notes on the IPv8 application -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Below is a brief description of some of the important classes in the IPv8 application (which are employed in the demo): - - -* - The ``RootEndpoint`` class will be the one which receives the requests from the peers. This class will delegate work to its children. Currently, the children are: ``AttestationEndpoint`` and ``NetworkEndpoint``. All these classes must subclass the BaseEndpoint class. The request chain is modeled like a tree, where the ``RootEndpoint`` is the root of the tree and the ``AttestationEndpoint`` and ``NetworkEndpoint`` classes are its children (and implicitly leaves) . When they are added, they are associated a ``path``\ , which is the ``endpoint`` field of the request (e.g. *attestation* or *network*\ ). This is how the root knows to delegate requests to the right children (e.g. ``:/``\ ). - -* - The ``NetworkEndpoint`` handles the *peer discovery* requests. It is (currently) a simple class which has the simple task of returning a list of verified nodes in the system and some relevant information on them, such as: *IP*\ , *port*\ , *public key*\ , and *offered services*. However, the Android Phone Application does not forward requests (GET or POST) to this endpoint. - -* - The ``AttestationEndpoint`` class handles a set of both **POST** and **GET** requests. It features a more complex set of methods than ``NetworkEndpoint``. - -* - The ``Response`` class is used to model HTTP responses. It will essentially just define a response (status_code, content) which will then be sent back as a response. - -* - The ``RESTManager`` class handles the start and stop of the server's HTTP API service. On startup, the class will create a ``RootEndpoint`` object, which will implicitly create the two objects: a ``NetworkEndpoint`` object and an ``AttestationEndpoint`` object. It will then attach itself to incoming request from ``localhost`` (\ ``127.0.0.1``\ ) at port ``8085``. To create a ``RestManager`` object, one must submit a *session* object, which is in fact of the ``IPv8`` class type (this latter instantiation requires a *configuration* object). - -Notes on the Python Scripts -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The **roles.py** script -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* - Holds the class ``TestRESTAPI`` which is a subclass of ``RESTManager``. It does the exact same thing as the ``RESTManager`` class, with the singular difference that it listens for incoming request on the ``8086`` port. - -* - The function ``sleep(time)`` is a simple NOOP function, which essentially forces the thread to sleep for a user specified amount of time (set by the parameter ``time``\ ), since it forces it to wait for a deferred to yield. - -* - The ``create_working_dir(path)`` function creates a working directory at the specified path, which will hold a number of **.pem** files, and an **sqlite** database (currently, for attestation and identity matters). - -* - The ``initialize_peer(path)`` function is used to create a peer (of type ``TestRESTAPI``\ ). It will generate a default peer configuration, which it will then modify to set the network overlays, which in this case is a *AttestationCommunity* and an *IdentityCommunity*. It will also set a peer discovery strategy within each of those overlays, which in this case is a *RandomWalk* with a limit of 20 peers and timeout of 60 seconds. A working directory is constructed, as described above (\ ``create_working_dir(*)``\ ). From this *configuration* an ``IPv8`` object is created, which is in turn used to create the peer, i.e. the ``TestRESTAPI``. This will also be assigned a (blank) HTML interface at ``URL = 127.0.0.1:8086/attestation``\ , that will define an access point to its REST API. The URL will be returned as the second object of the return statement. - -* - The ``stop_peers(*peers)`` function will simply do its best to stop the list of peers passed as parameter to it. - -* - The ``make_request(url, type, arguments={})`` function forwards a request of the specified ``type`` to the specified ``url``\ , with the specified set of ``arguments``. In this particular example, we usually have\ ``url = '127.0.0.1:8086/attestation'``\ , (\ **GET**\ ) (for **peers** and **outstanding** attestation requests) and (\ **POST**\ ) (for attesting outstanding requests). This function uses an ``Agent`` object, which models a simple HTTP client, that is able to generate HTTP requests given a set of parameters. Requests forwarded by this method are attached a callback, which is able to read the HTTP response body. - -The **main.py** script -~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -* - This script does not define any classes. It only defines a set of functions. - -* - This script creates a peer object by calling **roles.py**\ 's ``initialize_peer(path)``\ , with the ``path = "attester"``\ , and retrieves its URL towards accessing its REST API (\ ``idowner_restapi = 127.0.0.1:8086/attestation``\ ). - -* - The ``wait_for_peers(idowner_restapi)`` function will forward a (\ **GET**\ : *peers*\ ) request towards the ``idowner_restapi`` URL, i.e. the peer located behind it. The methods seeks to discover other peers in the network. It recursively calls itself every 4 seconds. Upon identification of at least one peer, it will stop calling itself, and return the list of discovered peers. - -* - The ``wait_for_attestation_request(attester_restapi)`` function will forward a (\ **GET**\ : *outstanding*\ ) towards the ``idowner_restapi`` URL, i.e. the peer located behind it. This methods seeks to find any, as of yet, unresolved *attestation* requests. It will recursively call itself every 4 seconds. Upon identification of at least one outstanding request, it will stop calling itself, and return the list of discovered requests. - -* - The ``make_attribute()`` method implements the main logic of the **main.py** script. It is triggered by a 500 millisecond deferred call . The method's flow is the following: It fist awaits for at least one fellow peer to show up by calling ``wait_for_peers(idowner_restapi)``\ , where the ``idowner_restapi`` is the URL of the ``TestRESTAPI`` object created as a global variable within the script. After this, it will await for at least one request to appear, by calling ``wait_for_attestation_request(attester_restapi)``\ , where the ``idowner_restapi`` is the URL of a ``TestRESTAPI`` object created as a global variable within the script. It will then solve each request by forwarding a (\ **POST**\ : *attest*\ ) with the requested attribute ("\ **QR**\ ") and a random encoded value associated to this attribute (since it is a demo) back to the ``TestRESTAPI`` object via its public URL. Once it finishes solving the requests it had initially found, it will end its loop and terminate its activity. - -Notes on the Android Application -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The **MainActivity** class -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following outlines the main behaviour of this class - - -* - Every second, the following happens: - - - * - The application will probe for peers. It does so by forwarding a (\ **GET**\ : *peers*\ ) to a well-known peer. If successful, the request will yield a response with an updated list of peers (represented by their unique IDs). Response timing is arbitrary, and cannot be anticipated clearly since it depends on network and other factors. Regardless, the peer will attempt to employ a local data structure, which is used for storing unique peer IDs (this might have been updated in the meantime, but it is not certain). If not possible to access the aforementioned list, it will return an empty list. It should be noted that when peer IDs are retrieved, the application will try to retrieve and store their attributes as well (by forwarding (\ **GET**\ : *attributes*\ ), with an additional request parameter ``mid = ``\ ). - - * - Next, it will attempt to request (and if the response arrives on time, update) the well-known peer's attributes (by forwarding a (\ **GET**\ : *attributes*\ ), **WITHOUT** the additional request parameter ``mid = ``\ ). Generally, however, this step refers to retrieving the local attributes list of the known peers (which should have been obtained as a collateral of the peer discovery process, as described above). If the list is not in use, it will return it, otherwise, an empty list is returned. If the list is empty, then the button, which demands attestation, is turned red, otherwise it is turned green. - -* - Asynchronously, the user may press the button on the interface, which will require that the 'QR' attribute be attested. If the application has discovered peers (i.e. at least one), then it will forward an attestation request (for this attribute) at every known peer in the network (by forwarding (\ **POST**\ : *request*\ ) HTTP requests, with additional request parameters ``mid = ``\ , and ``attribute_name = "QR"``\ ). Although attestation is requested from specific peers, in this demo implementation, any peer can solve them (see section **Side Note: Detailed Attestation Flow in the Demo**\ ). All requests are forwarded to the well-known peer. - -The **SingleShotRequest** class -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This class models a simple HTTP request, whose destination is hard-coded to be ``localhost:8085``. The constructor allows one to specify the endpoint (currently, we can have *attestation* or *network*\ , the latter not being used in the current Application implementation). It is also possible to specify the method (e.g. *POST*\ , *GET*\ , *PUT*\ , etc.), and a collection of (key, value) tuples, which act as the request's parameters. It should be noted that one specifies the nature of the request as part of the latter parameter, using the ``('type', )`` tuple (\ ```` can be *outstanding*\ , *verification_output*\ , *peers*\ , *attributes*\ , etc.). - -The **AttestationRESTInterface** class -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This class defines a set of public methods, which handle the creation and transmission of the different types of currently available HTTP requests. The following request types are defined (all of which are forwarded to the **attestation** endpoint): - - -* - **GET Method:** - - - * - ``retrieve_peers``\ : Forward a request for the peers in the system. Upon receiving a response, a callback is invoked to store the mid's of the discovered peers. This method also forwards requests for retrieving the peers' attributes, and relies of the callback to handle their storing. - - * - ``retrieve_outstanding``\ : Forward a request for the outstanding attestation requests in the system. Upon receiving a response, a callback is invoked to locally store the collection of discovered pending attestation requests. - - * - ``retrieve_verification_output``\ : Forward a request to get (all) the results of the verification processes. Upon receiving a response to the request, a callback is invoked which stores the verification results locally. - - * - ``retrieve_attributes(String mid)``\ : Forward a request for the attributes of a particular peer in the system, as identified by their mid. On response, invoke a callback, and store the attributes of the peer locally. - - * - ``retrieve_attributes(void)``\ : Forward a request for the attributes of the peer this request is sent to. On response, invoke a callback, which stores the attributes of the peer locally. - -* - **POST Method:** - - - * - ``put_request``\ : Forward a request which asks for attestation of a particular attribute from a particular peer. - - * - ``put_attest``\ : Attests a particular value of a particular attribute for a particular peer. - - * - ``put_verify``\ : Forward a request which asks for verification from a particular peer of a particular set of values associated to a particular attribute. diff --git a/doc/deprecated/attestation_tutorial.rst b/doc/deprecated/attestation_tutorial.rst deleted file mode 100644 index 677c17ff2..000000000 --- a/doc/deprecated/attestation_tutorial.rst +++ /dev/null @@ -1,173 +0,0 @@ - -Using the IPv8 attestation service -================================== - -This document assumes you have a basic understanding of network overlays in IPv8, as documented in `the overlay tutorial <../basics/overlay_tutorial.html>`_. -You will learn how to use the IPv8 attestation *HTTP REST API*. -This tutorial will use ``curl`` to perform HTTP ``GET`` and ``POST`` requests. - -Note that this tutorial will make use of the Python IPv8 service. -`An Android binding `_ is also available (\ `including demo app `_\ ). - -Running the IPv8 service ------------------------- - -Fill your ``main.py`` file with the following code (runnable with ``python3 main.py``\ ): - -.. literalinclude:: attestation_tutorial_1.py - -Running the service should yield something like the following output in your terminal: - -.. code-block:: bash - - $ python3 main.py - Starting peer aQVwz9aRMRypGwBkaxGRSdQs80c= - Starting peer bPyWPyswqXMhbW8+0RS6xUtNJrs= - -You should see two messages with 28 character base64 encoded strings. -These are the identifiers of the two peers we launched using the service. -You can use these identifiers for your reference when playing around with sending attestation requests. -In your experiment you will see other identifiers than the ``aQVwz9aRMRypGwBkaxGRSdQs80c=`` and ``bPyWPyswqXMhbW8+0RS6xUtNJrs=`` shown above. - -As a sanity check you can send your first HTTP ``GET`` requests and you should see that each peer can at least see the other peer. -Note that you might find more peers in the network. - -.. code-block:: bash - - $ curl http://localhost:14411/attestation?type=peers - ["bPyWPyswqXMhbW8+0RS6xUtNJrs="] - $ curl http://localhost:14412/attestation?type=peers - ["aQVwz9aRMRypGwBkaxGRSdQs80c="] - -Functionality flows -------------------- - -Generally speaking there are two (happy) flows when using the IPv8 attestation framework. -The first flow is the enrollment of an attribute and the second flow is the verification of an existing/enrolled attribute. -Both flows consist of a distinct set of requests (and responses) which we will explain in detail in the remainder of this document. - -To test a flow, we start the two peers we created previously. -If you did not remove the key files (\ ``*.pem``\ ) after the first run, you will start the same two peers as in the last run. -In our case the output of starting the service is as follows: - -.. code-block:: bash - - $ python main.py - Starting peer aQVwz9aRMRypGwBkaxGRSdQs80c= - Starting peer bPyWPyswqXMhbW8+0RS6xUtNJrs= - -In our case this means that peer ``aQVwz9aRMRypGwBkaxGRSdQs80c=`` exposes its REST API at ``http://localhost:14411/`` and peer ``bPyWPyswqXMhbW8+0RS6xUtNJrs=`` exposes its REST API at ``http://localhost:14412/``. -If you did not modify the ports in the initial scripts, you will have two different peer identifiers listening at the same ports. -For convenience we will refer to our first peer as *Peer 1* and our second peer as *Peer 2*. - -As a last note, beware of URL encoding: when passing these identifiers they need to be properly formatted (\ ``+`` and ``=`` are illegal characters). -In our case we need to use the following formatting of the peer identifiers in URLs (for Peer 1 and Peer 2 respectively): - -.. code-block:: console - - aQVwz9aRMRypGwBkaxGRSdQs80c%3D - bPyWPyswqXMhbW8%2B0RS6xUtNJrs%3D - -Enrollment/Attestation flow -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To enroll, or attest, an attribute we will go through the following steps: - - -#. Sanity checks: Peer 1 and Peer 2 can see each other and have no existing attributes. -#. Peer 1 requests attestation of an attribute by Peer 2. -#. Peer 2 attests to the requested attribute. -#. Peer 1 checks its attributes to confirm successful attestation. - -**0. SANITY CHECK -** First we check if both peers can see each other using their respective interfaces. - -.. code-block:: bash - - $ curl http://localhost:14411/attestation?type=peers - ["bPyWPyswqXMhbW8+0RS6xUtNJrs="] - $ curl http://localhost:14412/attestation?type=peers - ["aQVwz9aRMRypGwBkaxGRSdQs80c="] - -Then we confirm that neither peer has existing attributes. -Note that ``http://*:*/attestation?type=attributes`` is shorthand for ``http://*:*/attestation?type=attributes&mid=mid_b64`` where the identifier is equal to that of the calling peer. - -.. code-block:: bash - - $ curl http://localhost:14411/attestation?type=attributes - [] - $ curl http://localhost:14412/attestation?type=attributes - [] - -**1. ATTESTATION REQUEST -** Peer 1 will now ask Peer 2 to attest to an attribute. - -.. code-block:: bash - - $ curl -X POST "http://localhost:14411/attestation?type=request&mid=bPyWPyswqXMhbW8%2B0RS6xUtNJrs%3D&attribute_name=my_attribute" - -**2. ATTESTATION -** Peer 2 finds an outstanding request for attestation. -Peer 2 will now attest to some attribute value of Peer 1 (\ ``dmFsdWU%3D`` is the string ``value`` in base64 encoding). - -.. code-block:: bash - - $ curl http://localhost:14412/attestation?type=outstanding - [["aQVwz9aRMRypGwBkaxGRSdQs80c=", "my_attribute", "e30="]] - $ curl -X POST "http://localhost:14412/attestation?type=attest&mid=aQVwz9aRMRypGwBkaxGRSdQs80c%3D&attribute_name=my_attribute&attribute_value=dmFsdWU%3D" - -**3. CHECK -** Peer 1 confirms that he now has an attested attribute. - -.. code-block:: bash - - $ curl http://localhost:14411/attestation?type=attributes - [["my_attribute", "oEkkmxqu0Hd/aMVpSOdyP0SIlUM=", {"name": "my_attribute", "schema": "id_metadata", "date": 1592227939.021873}, "bPyWPyswqXMhbW8+0RS6xUtNJrs="]] - $ curl http://localhost:14412/attestation?type=attributes - [] - -Attribute verification flow -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To verify an attribute we will go through the following steps: - - -#. Sanity checks: Peer 1 and Peer 2 can see each other and Peer 1 has an existing attribute. -#. Peer 2 requests verification of an attribute of Peer 1. -#. Peer 1 allows verification of its attribute. -#. Peer 2 checks the verification output for the requested verification. - -**0. SANITY CHECK -** First we check if both peers can see each other using their respective interfaces. - -.. code-block:: bash - - $ curl http://localhost:14411/attestation?type=peers - ["bPyWPyswqXMhbW8+0RS6xUtNJrs="] - $ curl http://localhost:14412/attestation?type=peers - ["aQVwz9aRMRypGwBkaxGRSdQs80c="] - -Then we confirm that Peer 1 has the existing attribute (\ ``my_attribute`` from the last step). - -.. code-block:: bash - - $ curl http://localhost:14411/attestation?type=attributes - [["my_attribute", "oEkkmxqu0Hd/aMVpSOdyP0SIlUM=", {}, "bPyWPyswqXMhbW8+0RS6xUtNJrs="]] - $ curl http://localhost:14412/attestation?type=attributes - [] - -**1. VERIFICATION REQUEST -** Peer 2 will now ask Peer 1 to verify an attribute. - -.. code-block:: bash - - $ curl -X POST "http://localhost:14412/attestation?type=verify&mid=aQVwz9aRMRypGwBkaxGRSdQs80c%3D&attribute_hash=oEkkmxqu0Hd%2FaMVpSOdyP0SIlUM%3D&attribute_values=dmFsdWU%3D" - -**2. VERIFICATION -** Peer 1 finds an outstanding request for verification. - -.. code-block:: bash - - $ curl http://localhost:14411/attestation?type=outstanding_verify - [["bPyWPyswqXMhbW8+0RS6xUtNJrs=", "my_attribute"]] - $ curl -X POST "http://localhost:14411/attestation?type=allow_verify&mid=bPyWPyswqXMhbW8%2B0RS6xUtNJrs%3D&attribute_name=my_attribute" - -**3. CHECK -** Peer 2 checks the output of the verification process. - -.. code-block:: bash - - $ curl http://localhost:14412/attestation?type=verification_output - {"oEkkmxqu0Hd/aMVpSOdyP0SIlUM=": [["dmFsdWU=", 0.9999847412109375]]} diff --git a/doc/deprecated/attestation_tutorial_1.py b/doc/deprecated/attestation_tutorial_1.py deleted file mode 100644 index 770d3a5ac..000000000 --- a/doc/deprecated/attestation_tutorial_1.py +++ /dev/null @@ -1,46 +0,0 @@ -from asyncio import run -from base64 import b64encode - -from ipv8.configuration import get_default_configuration -from ipv8.REST.rest_manager import RESTManager -from ipv8.util import run_forever -from ipv8_service import IPv8 - - -async def start_communities() -> None: - # Launch two IPv8 services. - # We run REST endpoints for these services on: - # - http://localhost:14411/ - # - http://localhost:14412/ - # This script also prints the peer ids for reference with: - # - http://localhost:1441*/attestation?type=peers - for i in [1, 2]: - configuration = get_default_configuration() - configuration['logger']['level'] = "ERROR" - configuration['keys'] = [ - {'alias': "anonymous id", 'generation': "curve25519", 'file': "ec%d_multichain.pem" % i}, - ] - - # Only load the basic communities - requested_overlays = ['DiscoveryCommunity', 'AttestationCommunity', 'IdentityCommunity'] - configuration['overlays'] = [o for o in configuration['overlays'] if o['class'] in requested_overlays] - - # Give each peer a separate working directory - working_directory_overlays = ['AttestationCommunity', 'IdentityCommunity'] - for overlay in configuration['overlays']: - if overlay['class'] in working_directory_overlays: - overlay['initialize'] = {'working_directory': 'state_%d' % i} - - # Start the IPv8 service - ipv8 = IPv8(configuration) - await ipv8.start() - rest_manager = RESTManager(ipv8) - await rest_manager.start(14410 + i) - - # Print the peer for reference - print("Starting peer", b64encode(ipv8.keys["anonymous id"].mid)) - - await run_forever() - - -run(start_communities()) diff --git a/doc/deprecated/attestation_tutorial_integration/__init__.py b/doc/deprecated/attestation_tutorial_integration/__init__.py deleted file mode 100644 index d674afa65..000000000 --- a/doc/deprecated/attestation_tutorial_integration/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -1. Run ``attestation_tutorial_attest``. -2. Run ``attestation_tutorial_verify``. - -May freeze in case of horrible error: be sure to kill scripts if they run longer than ~1 minute. -""" diff --git a/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_attest.py b/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_attest.py deleted file mode 100644 index 55c86990d..000000000 --- a/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_attest.py +++ /dev/null @@ -1,49 +0,0 @@ -import os -import shutil - -from attestation_tutorial_common import finish, http_get, http_post, start, urlstr, wait_for_list - -# Remove the output of previous experiments. -if os.path.exists('./state_1'): - shutil.rmtree('./state_1') -if os.path.exists('./state_2'): - shutil.rmtree('./state_2') - -start() -print("Enrollment/Attestation flow") - -print("0. SANITY CHECK") -peer1_neighborhood = wait_for_list("http://localhost:14411/attestation?type=peers") -peer2_neighborhood = wait_for_list("http://localhost:14412/attestation?type=peers") - -peer1_id = urlstr(peer2_neighborhood[0]) -peer2_id = urlstr(peer1_neighborhood[0]) - -print("Peer 1:", peer1_id) -print("Peer 2:", peer2_id) - -print("Peer 1 attributes:", http_get("http://localhost:14411/attestation?type=attributes")) -print("Peer 2 attributes:", http_get("http://localhost:14412/attestation?type=attributes")) - -print("1. ATTESTATION REQUEST") -print("Request attestation from peer 2:", - http_post(f"http://localhost:14411/attestation?type=request&mid={peer2_id}&attribute_name=my_attribute")) - -print("2. ATTESTATION") -peer2_outstanding_requests = wait_for_list("http://localhost:14412/attestation?type=outstanding") -print("Peer 2 outstanding requests:", peer2_outstanding_requests) - -print("Peer 2 attesting to outstanding request:", - http_post(f"http://localhost:14412/attestation?type=attest&mid={peer1_id}" - f"&attribute_name={peer2_outstanding_requests[0][1]}" - f"&attribute_value=dmFsdWU%3D")) - -print("3. CHECK") -peer1_attributes = http_get("http://localhost:14411/attestation?type=attributes") -print("Peer 1 attributes:", peer1_attributes) -print("Peer 2 attributes:", http_get("http://localhost:14412/attestation?type=attributes")) - -assert len(peer1_attributes) > 0 - -print("X. DONE!") -finish() diff --git a/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_common.py b/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_common.py deleted file mode 100644 index cc491ac8c..000000000 --- a/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_common.py +++ /dev/null @@ -1,66 +0,0 @@ -import binascii -import json -import os -import signal -import subprocess -import time -import urllib.parse -import urllib.request -from typing import Any, cast - -PROCESS = None -ID1 = binascii.hexlify(os.urandom(20)).decode() -ID2 = binascii.hexlify(os.urandom(20)).decode() -ID3 = binascii.hexlify(os.urandom(20)).decode() - - -def http_get(url: str) -> Any: - """ - Perform an HTTP GET request to the given URL. - """ - return json.loads(urllib.request.urlopen(urllib.request.Request(url)).read().decode()) # noqa: S310 - - -def http_post(url: str) -> Any: - """ - Perform an HTTP POST request to the given URL. - """ - return json.loads(urllib.request.urlopen(urllib.request.Request(url, method="POST")).read().decode()) # noqa: S310 - - -def urlstr(s: str) -> str: - """ - Make the given string URL safe. - """ - return urllib.parse.quote(s, safe='') - - -def wait_for_list(url: str) -> list: - """ - Poll an endpoint until output (a list) is available. - """ - out = [] - while not out: - out = http_get(url) - time.sleep(0.5) - return out - - -def start() -> None: - """ - Run the main.py script and wait for it to finish initializing. - """ - global PROCESS # noqa: PLW0603 - PROCESS = subprocess.Popen(f'python3 main.py {ID1} {ID2} {ID3}', - shell=True, preexec_fn=os.setsid) # noqa: PLW1509,S602 - os.waitpid(PROCESS.pid, os.P_NOWAITO) - time.sleep(5.0) - - -def finish() -> None: - """ - Kill our two IPv8 instances (running in the same process). - """ - process = cast(subprocess.Popen, PROCESS) - os.killpg(os.getpgid(process.pid), signal.SIGTERM) - process.communicate() diff --git a/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_verify.py b/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_verify.py deleted file mode 100644 index 6392d2499..000000000 --- a/doc/deprecated/attestation_tutorial_integration/attestation_tutorial_verify.py +++ /dev/null @@ -1,48 +0,0 @@ -import time - -from attestation_tutorial_common import finish, http_get, http_post, start, urlstr, wait_for_list - -start() -print("Attribute verification flow") - -print("0. SANITY CHECK") -peer1_neighborhood = wait_for_list("http://localhost:14411/attestation?type=peers") -peer2_neighborhood = wait_for_list("http://localhost:14412/attestation?type=peers") - -peer1_id = urlstr(peer2_neighborhood[0]) -peer2_id = urlstr(peer1_neighborhood[0]) - -print("Peer 1:", peer1_id) -print("Peer 2:", peer2_id) - -peer1_attributes = http_get("http://localhost:14411/attestation?type=attributes") -peer2_attributes = http_get("http://localhost:14412/attestation?type=attributes") - -print("Peer 1 attributes:", peer1_attributes) -print("Peer 2 attributes:", peer2_attributes) - -print("1. VERIFICATION REQUEST") -print("Request verification from peer 1:", - http_post(f"http://localhost:14412/attestation?type=verify&mid={peer1_id}" - f"&attribute_hash={urlstr(peer1_attributes[-1][1])}&attribute_values=dmFsdWU%3D")) - -print("2. VERIFICATION ") -peer1_outstanding_requests = wait_for_list("http://localhost:14411/attestation?type=outstanding_verify") -print("Peer 1 outstanding verification requests:", peer1_outstanding_requests) - -print("Peer 1 allow verification of outstanding request:", - http_post(f"http://localhost:14411/attestation?type=allow_verify&mid={peer2_id}" - f"&attribute_name={urlstr(peer1_attributes[-1][0])}")) - -print("3. CHECK") -match = 0.0 -while match < 0.9: - for attribute_hash, output in http_get("http://localhost:14412/attestation?type=verification_output").items(): - if attribute_hash == peer1_attributes[-1][1]: - match_value, match = output[0] - assert match_value == "dmFsdWU=" - time.sleep(0.1) -print("Peer 2 verification output:", http_get("http://localhost:14412/attestation?type=verification_output")) - -print("X. DONE!") -finish() diff --git a/doc/deprecated/attestation_tutorial_integration/main.py b/doc/deprecated/attestation_tutorial_integration/main.py deleted file mode 100644 index 459ad4534..000000000 --- a/doc/deprecated/attestation_tutorial_integration/main.py +++ /dev/null @@ -1,129 +0,0 @@ -from asyncio import run -from base64 import b64encode -from binascii import unhexlify -from sys import argv -from time import sleep - -from ipv8.attestation.identity.community import IdentityCommunity -from ipv8.attestation.wallet.community import AttestationCommunity -from ipv8.configuration import DISPERSY_BOOTSTRAPPER, get_default_configuration -from ipv8.peerdiscovery.community import DiscoveryCommunity -from ipv8.REST.rest_manager import RESTManager -from ipv8.util import run_forever -from ipv8_service import IPv8 - - -class IsolatedIdentityCommunity(IdentityCommunity): - community_id = unhexlify(argv[1]) - - -class IsolatedAttestationCommunity(AttestationCommunity): - community_id = unhexlify(argv[2]) - - -class IsolatedDiscoveryCommunity(DiscoveryCommunity): - community_id = unhexlify(argv[3]) - - -async def start_communities() -> None: - # Launch two IPv8 services. - # We run REST endpoints for these services on: - # - http://localhost:14411/ - # - http://localhost:14412/ - # This script also prints the peer ids for reference with: - # - http://localhost:1441*/attestation?type=peers - for i in [1, 2]: - configuration = get_default_configuration() - configuration['logger']['level'] = "ERROR" - configuration['keys'] = [ - {'alias': "anonymous id", 'generation': "curve25519", 'file': "ec%d_multichain.pem" % i} - ] - - # Only load the basic communities - configuration['overlays'] = [ - { - 'class': 'IsolatedDiscoveryCommunity', - 'key': "anonymous id", - 'walkers': [ - { - 'strategy': "RandomWalk", - 'peers': 20, - 'init': { - 'timeout': 3.0 - } - }, - { - 'strategy': "RandomChurn", - 'peers': -1, - 'init': { - 'sample_size': 8, - 'ping_interval': 10.0, - 'inactive_time': 27.5, - 'drop_time': 57.5 - } - }, - { - 'strategy': "PeriodicSimilarity", - 'peers': -1, - 'init': {} - } - ], - 'bootstrappers': [DISPERSY_BOOTSTRAPPER], - 'initialize': {}, - 'on_start': [] - }, - { - 'class': 'IsolatedAttestationCommunity', - 'key': "anonymous id", - 'walkers': [{ - 'strategy': "RandomWalk", - 'peers': 20, - 'init': { - 'timeout': 3.0 - } - }], - 'bootstrappers': [DISPERSY_BOOTSTRAPPER], - 'initialize': {'working_directory': 'state_%d' % i}, - 'on_start': [] - }, - { - 'class': 'IsolatedIdentityCommunity', - 'key': "anonymous id", - 'walkers': [{ - 'strategy': "RandomWalk", - 'peers': 20, - 'init': { - 'timeout': 3.0 - } - }], - 'bootstrappers': [DISPERSY_BOOTSTRAPPER], - 'initialize': {'working_directory': 'state_%d' % i}, - 'on_start': [] - } - ] - - # Start the IPv8 service - ipv8 = IPv8(configuration, extra_communities={ - 'IsolatedDiscoveryCommunity': IsolatedDiscoveryCommunity, - 'IsolatedAttestationCommunity': IsolatedAttestationCommunity, - 'IsolatedIdentityCommunity': IsolatedIdentityCommunity - }) - await ipv8.start() - rest_manager = RESTManager(ipv8) - - # We REALLY want this particular port, keep trying - keep_trying = True - while keep_trying: - try: - await rest_manager.start(14410 + i) - keep_trying = False - except OSError: - sleep(1.0) # noqa: ASYNC101 - - # Print the peer for reference - print("Starting peer", b64encode(ipv8.keys["anonymous id"].mid)) - - await run_forever() - - -run(start_communities()) diff --git a/doc/deprecated/resources/android_rest_api.png b/doc/deprecated/resources/android_rest_api.png deleted file mode 100644 index 9b3b39f0b..000000000 Binary files a/doc/deprecated/resources/android_rest_api.png and /dev/null differ diff --git a/doc/deprecated/resources/attestation_req.png b/doc/deprecated/resources/attestation_req.png deleted file mode 100644 index 05eefde27..000000000 Binary files a/doc/deprecated/resources/attestation_req.png and /dev/null differ diff --git a/doc/deprecated/resources/output_SQjlvW.gif b/doc/deprecated/resources/output_SQjlvW.gif deleted file mode 100644 index 0ef0268f0..000000000 Binary files a/doc/deprecated/resources/output_SQjlvW.gif and /dev/null differ diff --git a/doc/deprecated/resources/peer_rest_api.png b/doc/deprecated/resources/peer_rest_api.png deleted file mode 100644 index a637e5b6f..000000000 Binary files a/doc/deprecated/resources/peer_rest_api.png and /dev/null differ diff --git a/ipv8/REST/asyncio_endpoint.py b/ipv8/REST/asyncio_endpoint.py index fe5bfa54f..dba35a167 100644 --- a/ipv8/REST/asyncio_endpoint.py +++ b/ipv8/REST/asyncio_endpoint.py @@ -4,7 +4,8 @@ import logging import time from asyncio import all_tasks, current_task, get_running_loop -from typing import TYPE_CHECKING, Iterable, cast +from collections.abc import Iterable +from typing import TYPE_CHECKING, cast from aiohttp import web from aiohttp_apispec import docs, json_schema @@ -193,8 +194,7 @@ async def get_asyncio_tasks(self, _: Request) -> Response: current = current_task() tasks = [] for task in all_tasks(): - # Only in Python 3.8+ will we have a get_name function - name = task.get_name() if hasattr(task, 'get_name') else getattr(task, 'name', f'Task-{id(task)}') + name = task.get_name() task_dict = {"name": name, "running": task == current, diff --git a/ipv8/REST/attestation_endpoint.py b/ipv8/REST/attestation_endpoint.py deleted file mode 100644 index 43d222f78..000000000 --- a/ipv8/REST/attestation_endpoint.py +++ /dev/null @@ -1,373 +0,0 @@ -from __future__ import annotations - -import json -from asyncio import Future -from base64 import b64decode, b64encode -from hashlib import sha1 -from typing import TYPE_CHECKING, cast - -from aiohttp import web -from aiohttp_apispec import docs - -from ..attestation.identity.community import IdentityCommunity, create_community -from ..attestation.wallet.community import AttestationCommunity -from ..keyvault.crypto import default_eccrypto -from ..peer import Peer -from ..types import IPv8, PrivateKey -from ..util import strip_sha1_padding, succeed -from .base_endpoint import HTTP_BAD_REQUEST, HTTP_NOT_FOUND, BaseEndpoint, Response - -if TYPE_CHECKING: - from aiohttp.abc import Request - - -class AttestationEndpoint(BaseEndpoint[IPv8]): - """ - This endpoint is responsible for handing all requests regarding attestation. - """ - - def __init__(self) -> None: - """ - Create new unregistered and uninitialized REST endpoint. - """ - super().__init__() - self.attestation_overlay: AttestationCommunity | None = None - self.identity_overlay: IdentityCommunity | None = None - self.persistent_key: Peer | None = None - self.attestation_requests: dict[tuple[str, str], tuple[Future, str]] = {} - self.verify_requests: dict[tuple[str, str], Future] = {} - self.verification_output: dict[bytes, list[tuple[bytes, float]]] = {} - self.attestation_metadata: dict[tuple[Peer, str], dict] = {} - - def setup_routes(self) -> None: - """ - Register the names to make this endpoint callable. - """ - self.app.add_routes([web.get('', self.handle_get), - web.post('', self.handle_post)]) - - def initialize(self, session: IPv8) -> None: - """ - Initialize this endpoint for the given IPv8 instance. - """ - super().initialize(session) - self.attestation_overlay = next((overlay for overlay in session.overlays - if isinstance(overlay, AttestationCommunity)), None) - self.identity_overlay = next((overlay for overlay in session.overlays - if isinstance(overlay, IdentityCommunity)), None) - if self.attestation_overlay and self.identity_overlay: - self.attestation_overlay.set_attestation_request_callback(self.on_request_attestation) - self.attestation_overlay.set_attestation_request_complete_callback(self.on_attestation_complete) - self.attestation_overlay.set_verify_request_callback(self.on_verify_request) - self.persistent_key = self.identity_overlay.my_peer - - def on_request_attestation(self, peer: Peer, attribute_name: str, metadata: dict) -> Future: - """ - Return the measurement of an attribute for a certain peer. - """ - future: Future = Future() - self.attestation_requests[(b64encode(peer.mid).decode(), attribute_name)] = \ - (future, b64encode(json.dumps(metadata).encode('utf-8')).decode()) - self.attestation_metadata[(peer, attribute_name)] = metadata - return future - - def on_attestation_complete(self, for_peer: Peer, attribute_name: str, attribute_hash: bytes, - id_format: str, from_peer: Peer | None = None) -> None: - """ - Callback for when an attestation has been completed for another peer. - We can now sign for it. - """ - self.identity_overlay = cast(IdentityCommunity, self.identity_overlay) - metadata = self.attestation_metadata.get((for_peer, attribute_name), None) - if for_peer.mid == self.identity_overlay.my_peer.mid: - from_peer = cast(Peer, from_peer) - if from_peer.mid == self.identity_overlay.my_peer.mid: - self.identity_overlay.self_advertise(attribute_hash, attribute_name, id_format, metadata) - else: - self.identity_overlay.request_attestation_advertisement(from_peer, attribute_hash, attribute_name, - id_format, metadata) - else: - self.identity_overlay.add_known_hash(attribute_hash, attribute_name, for_peer.public_key.key_to_bin(), - metadata) - - def on_verify_request(self, peer: Peer, attribute_hash: bytes) -> Future: - """ - Return the measurement of an attribute for a certain peer. - """ - self.identity_overlay = cast(IdentityCommunity, self.identity_overlay) - metadata = self.identity_overlay.get_attestation_by_hash(attribute_hash) - if not metadata: - return succeed(None) - attribute_name = json.loads(metadata.serialized_json_dict)["name"] - future: Future = Future() - self.verify_requests[(b64encode(peer.mid).decode(), attribute_name)] = future - return future - - def on_verification_results(self, attribute_hash: bytes, values: list[float]) -> None: - """ - Callback for when verification has concluded. - """ - references = self.verification_output[attribute_hash] - out = [(cast(bytes, references[i][0]) if isinstance(references[i], tuple) else cast(bytes, references[i]), - values[i]) - for i in range(len(references))] - self.verification_output[attribute_hash] = out - - def get_peer_from_mid(self, mid_b64: str) -> Peer | None: - """ - Find a peer by base64 encoded mid. - """ - if self.session is None: - return None - mid = b64decode(mid_b64) - peers = self.session.network.verified_peers - matches = [p for p in peers if p.mid == mid] - return matches[0] if matches else None - - def _drop_identity_table_data(self, keys_to_keep: list[bytes]) -> list[bytes]: - """ - Remove all metadata from the identity community. - - :param keys_to_keep: list of keys to not remove for - :return: the list of attestation hashes which have been removed - """ - self.identity_overlay = cast(IdentityCommunity, self.identity_overlay) - database = self.identity_overlay.identity_manager.database - all_identities = database.get_known_identities() - to_remove = [] - attestation_hashes = [] - for key in all_identities: - if key not in keys_to_keep: - to_remove.append(key) - attestation_hashes.extend([t.content_hash for t in database.get_tokens_for(Peer(key).public_key)]) - - with database: - for public_key in to_remove: - database.executescript("BEGIN TRANSACTION; " - "DELETE FROM Tokens WHERE public_key = ?; " - "DELETE FROM Metadata WHERE public_key = ?; " - "DELETE FROM Attestations WHERE public_key = ?; " - "COMMIT;", - (public_key, public_key, public_key)) - database.commit() - - return attestation_hashes - - def _drop_attestation_table_data(self, attestation_hashes: list[bytes]) -> None: - """ - Remove all attestation data (claim based keys and ZKP blobs) by list of attestation hashes. - - :param attestation_hashes: hashes to remove - :returns: None - """ - if not attestation_hashes: - return - self.attestation_overlay = cast(AttestationCommunity, self.attestation_overlay) - - self.attestation_overlay.database.execute(("DELETE FROM %s" # noqa: S608 - % self.attestation_overlay.database.db_name) - + " WHERE hash IN (" - + ", ".join(c for c in "?" * len(attestation_hashes)) - + ")", - attestation_hashes) - self.attestation_overlay.database.commit() - - @docs( - tags=["Attestation"], - summary="Get information from the AttestationCommunity.", - parameters=[{ - 'in': 'query', - 'name': 'type', - 'description': 'Type of query', - 'type': 'string', - 'enum': ['drop_identity', 'outstanding', 'outstanding_verify', 'verification_output', 'peers', 'attributes'], - 'required': True - }, { - 'in': 'query', - 'name': 'mid', - 'description': 'Filter by mid (only works for type=attributes)', - 'type': 'string' - }], - description=""" - type=drop_identity - type=outstanding -> [(mid_b64, attribute_name)] - type=outstanding_verify -> [(mid_b64, attribute_name)] - type=verification_output -> {hash_b64: [(value_b64, match)]} - type=peers -> [mid_b64] - type=attributes&mid=mid_b64 -> [(attribute_name, attribute_hash)] - """ - ) - async def handle_get(self, request: Request) -> Response: # noqa: C901, PLR0911, PLR0912 - """ - Get information from the AttestationCommunity. - """ - if self.session is None or self.attestation_overlay is None or self.identity_overlay is None: - return Response({"error": "attestation or identity community not found"}, status=HTTP_NOT_FOUND) - self.session = cast(IPv8, self.session) - self.attestation_overlay = cast(AttestationCommunity, self.attestation_overlay) - self.identity_overlay = cast(IdentityCommunity, self.identity_overlay) - - if not request.query or 'type' not in request.query: - return Response({"error": "parameters or type missing"}, status=HTTP_BAD_REQUEST) - - if request.query['type'] == 'outstanding': - return Response([(*k, v[1]) for k, v in self.attestation_requests.items()]) - - if request.query['type'] == 'outstanding_verify': - return Response(list(self.verify_requests)) - - if request.query['type'] == 'verification_output': - formatted_vfo = {} - for k, v in self.verification_output.items(): - formatted_vfo[b64encode(k).decode('utf-8')] = [(b64encode(a).decode('utf-8'), m) for a, m in v] - return Response(formatted_vfo) - - if request.query['type'] == 'peers': - peers = self.session.network.get_peers_for_service(self.identity_overlay.community_id) - return Response([b64encode(p.mid).decode('utf-8') for p in peers]) - - if request.query['type'] == 'attributes': - if 'mid' in request.query: - mid_b64 = request.query['mid'] - peer = self.get_peer_from_mid(mid_b64) - else: - peer = self.identity_overlay.my_peer - if peer: - pseudonym = self.identity_overlay.identity_manager.get_pseudonym(peer.public_key) - trimmed = {} - for credential in pseudonym.get_credentials(): - # TODO: add support for more attesters # noqa: FIX002, TD002, TD003 - attestations = list(credential.attestations) - if attestations: - authority = self.identity_overlay.identity_manager.database.get_authority(attestations[0]) - attester = b64encode(sha1(authority).digest()).decode() - else: - attester = "" - attribute_hash = pseudonym.tree.elements[credential.metadata.token_pointer].content_hash - json_metadata = json.loads(credential.metadata.serialized_json_dict) - trimmed[attribute_hash] = (json_metadata["name"], json_metadata, attester) - # List of (name, attribute_hash, metadata, attester) - return Response([( - data[0], - b64encode(strip_sha1_padding(attribute_hash)).decode(), - data[1], - data[2]) for attribute_hash, data in trimmed.items()]) - return Response([]) - - if request.query['type'] == 'drop_identity': - self.session = cast(IPv8, self.session) - self.persistent_key = cast(Peer, self.persistent_key) - - to_keep = [self.persistent_key.public_key.key_to_bin()] - if 'keep' in request.query: - to_keep += [self.identity_overlay.my_peer.public_key.key_to_bin()] - - # Remove identity metadata and attestation proofing data, except for the keys to keep - attestation_hashes = self._drop_identity_table_data(to_keep) - self._drop_attestation_table_data(attestation_hashes) - - # Remove pending attestations - self.attestation_requests.clear() - - # Generate new key - my_new_peer = Peer(default_eccrypto.generate_key("curve25519")) - identity_manager = self.identity_overlay.identity_manager - await self.session.unload_overlay(self.identity_overlay) - self.identity_overlay = await create_community(cast(PrivateKey, my_new_peer.key), self.session, - identity_manager, endpoint=self.session.endpoint) - for overlay in self.session.overlays: - overlay.my_peer = my_new_peer - return Response({"success": True}) - - return Response({"error": "type argument incorrect"}, status=HTTP_BAD_REQUEST) - - @docs( - tags=["Attestation"], - summary="Send a command to the AttestationCommunity.", - parameters=[{ - 'in': 'query', - 'name': 'type', - 'description': 'Type of query', - 'type': 'string', - 'enum': ['request', 'allow_verify', 'attest', 'verify'], - 'required': True - }], - description=""" - type=request&mid=mid_b64&attibute_name=attribute_name&id_format=id_format - type=allow_verify&mid=mid_b64&attibute_name=attribute_name - type=attest&mid=mid_b64&attribute_name=attribute_name&attribute_value=attribute_value_b64 - type=verify&mid=mid_b64&attribute_hash=attribute_hash_b64&id_format=id_format - &attribute_values=attribute_value_b64,... - """ - ) - async def handle_post(self, request: Request) -> Response: # noqa: C901, PLR0911 - """ - Send a command to the AttestationCommunity. - """ - if not self.attestation_overlay or not self.identity_overlay: - return Response({"error": "attestation or identity community not found"}, status=HTTP_NOT_FOUND) - - args = request.query - if not args or 'type' not in args: - return Response({"error": "parameters or type missing"}, status=HTTP_BAD_REQUEST) - - if args['type'] == 'request': - mid_b64 = args['mid'] - attribute_name = args['attribute_name'] - id_format = args.get('id_format', 'id_metadata') - peer = self.get_peer_from_mid(mid_b64) - if peer: - key = self.attestation_overlay.get_id_algorithm(id_format).generate_secret_key() - metadata = {"id_format": id_format} - if 'metadata' in args: - metadata_unicode = json.loads(b64decode(args['metadata'])) - metadata.update(metadata_unicode) - self.attestation_metadata[(self.identity_overlay.my_peer, attribute_name)] = metadata - self.attestation_overlay.request_attestation(peer, attribute_name, key, metadata) - return Response({"success": True}) - return Response({"error": "peer unknown"}, status=HTTP_BAD_REQUEST) - - if args['type'] == 'attest': - mid_b64 = args['mid'] - attribute_name = args['attribute_name'] - attribute_value_b64 = args['attribute_value'] - outstanding = self.attestation_requests.pop((mid_b64, attribute_name)) - outstanding[0].set_result(b64decode(attribute_value_b64)) - return Response({"success": True}) - - if args['type'] == 'import_blob': - # Import self-attested binary data - attribute_name = args['attribute_name'] - id_format = args['id_format'] - metadata = {"id_format": id_format} - if 'metadata' in args: - metadata_unicode = json.loads(b64decode(args['metadata'])) - for k, v in metadata_unicode.items(): - metadata[k.encode()] = v.encode() - blob = await request.read() - - self.attestation_overlay.dump_blob(attribute_name, id_format, blob, metadata) - - return Response({"success": True}) - - if args['type'] == 'allow_verify': - mid_b64 = args['mid'] - attribute_name = args['attribute_name'] - self.verify_requests.pop((mid_b64, attribute_name)).set_result(True) - return Response({"success": True}) - - if args['type'] == 'verify': - mid_b64 = args['mid'] - attribute_hash = b64decode(args['attribute_hash']) - reference_values = [b64decode(v) for v in args['attribute_values'].split(',')] - id_format = args.get('id_format', 'id_metadata') - peer = self.get_peer_from_mid(mid_b64) - if peer: - self.verification_output[attribute_hash] = \ - [(b64decode(v), 0.0) for v in args['attribute_values'].split(',')] - self.attestation_overlay.verify_attestation_values(peer.address, attribute_hash, reference_values, - self.on_verification_results, id_format) - return Response({"success": True}) - return Response({"error": "peer unknown"}, status=HTTP_BAD_REQUEST) - - return Response({"error": "type argument incorrect"}, status=HTTP_BAD_REQUEST) diff --git a/ipv8/REST/base_endpoint.py b/ipv8/REST/base_endpoint.py index 734627599..d8f36107d 100644 --- a/ipv8/REST/base_endpoint.py +++ b/ipv8/REST/base_endpoint.py @@ -2,7 +2,8 @@ import json import logging -from typing import Any, Awaitable, Callable, Generic, Iterable, TypeVar +from collections.abc import Awaitable, Iterable +from typing import Any, Callable, Generic, TypeVar from aiohttp import web from aiohttp.abc import Request, StreamResponse diff --git a/ipv8/REST/dht_endpoint.py b/ipv8/REST/dht_endpoint.py index 8bf5c1fbc..0511268a5 100644 --- a/ipv8/REST/dht_endpoint.py +++ b/ipv8/REST/dht_endpoint.py @@ -1,9 +1,12 @@ from __future__ import annotations +import functools +import operator from base64 import b64encode from binascii import hexlify, unhexlify +from collections.abc import Sequence from timeit import default_timer -from typing import TYPE_CHECKING, List, Optional, Sequence, Tuple, cast +from typing import TYPE_CHECKING, Optional, cast from aiohttp import web from aiohttp_apispec import docs, json_schema @@ -95,7 +98,7 @@ async def get_statistics(self, _: Request) -> Response: for address_cls, routing_table in self.dht.routing_tables.items(): buckets = routing_table.trie.values() address = self.dht.my_peer.addresses.get(address_cls, self.dht.my_estimated_wan) - endpoints = cast(List, stats["endpoints"]) + endpoints = cast(list, stats["endpoints"]) endpoints.append({ "endpoint": FAST_ADDR_TO_INTERFACE[address_cls], "node_id": hexlify(calc_node_id(address, self.dht.my_peer.mid)).decode('utf-8'), @@ -209,10 +212,10 @@ async def get_values(self, request: Request) -> Response: key = unhexlify(request.match_info['key']) start = default_timer() - values, crawls = cast(Tuple[Sequence[Tuple[bytes, Optional[bytes]]], List[Crawl]], + values, crawls = cast(tuple[Sequence[tuple[bytes, Optional[bytes]]], list[Crawl]], await self.dht.find_values(key, debug=True)) nodes_tried = set().union(*[crawl.nodes_tried for crawl in crawls]) - responses: list[tuple[Node, dict]] = sum([crawl.responses for crawl in crawls], []) + responses: list[tuple[Node, dict]] = functools.reduce(operator.iadd, [crawl.responses for crawl in crawls], []) stop = default_timer() return Response({ diff --git a/ipv8/REST/identity_endpoint.py b/ipv8/REST/identity_endpoint.py index eb777e8ad..01b60423d 100644 --- a/ipv8/REST/identity_endpoint.py +++ b/ipv8/REST/identity_endpoint.py @@ -496,7 +496,7 @@ async def create_pseudonym_credential(self, request: Request) -> Response: if authority is None: return Response({"success": False, "error": "failed to find authority"}) - metadata = parameters['metadata'] if 'metadata' in parameters else {} + metadata = parameters.get("metadata", {}) channel.request_attestation(authority, parameters['name'], parameters['schema'], metadata) return Response({"success": True}) diff --git a/ipv8/REST/root_endpoint.py b/ipv8/REST/root_endpoint.py index cd0705f96..e84aef09c 100644 --- a/ipv8/REST/root_endpoint.py +++ b/ipv8/REST/root_endpoint.py @@ -1,5 +1,4 @@ from .asyncio_endpoint import AsyncioEndpoint -from .attestation_endpoint import AttestationEndpoint from .base_endpoint import BaseEndpoint from .dht_endpoint import DHTEndpoint from .identity_endpoint import IdentityEndpoint @@ -21,7 +20,6 @@ def setup_routes(self) -> None: Register the names to make this endpoint callable. """ endpoints = {'/asyncio': AsyncioEndpoint, - '/attestation': AttestationEndpoint, '/dht': DHTEndpoint, '/identity': IdentityEndpoint, '/isolation': IsolationEndpoint, diff --git a/ipv8/REST/schema.py b/ipv8/REST/schema.py index ae2991d63..5f5b988cb 100644 --- a/ipv8/REST/schema.py +++ b/ipv8/REST/schema.py @@ -60,7 +60,7 @@ class OverlaySchema(Schema): The schema to describe overlays. """ - id = String() # noqa: A003 + id = String() my_peer = String() global_time = Integer() peers = List(Nested(cast(SchemaABC, AddressWithPK))) diff --git a/ipv8/attestation/communication_manager.py b/ipv8/attestation/communication_manager.py index a3fb59aae..67e15cdc4 100644 --- a/ipv8/attestation/communication_manager.py +++ b/ipv8/attestation/communication_manager.py @@ -4,7 +4,7 @@ import base64 import json import os -from typing import Dict, Tuple, cast +from typing import cast from ..keyvault.crypto import ECCrypto from ..messaging.anonymization.endpoint import TunnelEndpoint @@ -15,8 +15,8 @@ from .identity.manager import IdentityManager from .wallet.community import AttestationCommunity, AttestationSettings -AttributePointer = Tuple[Peer, str] # Backward compatibility: Python >= 3.9 can use ``tuple[Peer, str]`` -MetadataDict = Dict[str, str] # Backward compatibility: Python >= 3.9 can use ``dict[str, str]`` +AttributePointer = tuple[Peer, str] +MetadataDict = dict[str, str] class CommunicationChannel: diff --git a/ipv8/attestation/identity/database.py b/ipv8/attestation/identity/database.py index a3214c320..5ec0c84c9 100644 --- a/ipv8/attestation/identity/database.py +++ b/ipv8/attestation/identity/database.py @@ -16,7 +16,7 @@ class Credential: Cache for Metadata <- [Attestation] mappings. """ - def __init__(self, metadata: Metadata, attestations: typing.Set[Attestation]) -> None: + def __init__(self, metadata: Metadata, attestations: set[Attestation]) -> None: """ Create a new credential. """ @@ -75,7 +75,7 @@ def insert_attestation(self, public_key: PublicKey, authority_key: PublicKey, at (public_key.key_to_bin(), authority_key.key_to_bin(), metadata_pointer, signature)) self.commit() - def get_tokens_for(self, public_key: PublicKey) -> typing.Set[Token]: + def get_tokens_for(self, public_key: PublicKey) -> set[Token]: """ Get all tokens in the tree of a certain public key. """ @@ -84,7 +84,7 @@ def get_tokens_for(self, public_key: PublicKey) -> typing.Set[Token]: fetch_all=True)) return {Token.from_database_tuple(*token) for token in tokens} - def get_metadata_for(self, public_key: PublicKey) -> typing.Set[Metadata]: + def get_metadata_for(self, public_key: PublicKey) -> set[Metadata]: """ Get all known metadata for a certain public key. """ @@ -93,7 +93,7 @@ def get_metadata_for(self, public_key: PublicKey) -> typing.Set[Metadata]: fetch_all=True)) return {Metadata.from_database_tuple(*metadato) for metadato in metadata} - def get_attestations_for(self, public_key: PublicKey) -> typing.Set[Attestation]: + def get_attestations_for(self, public_key: PublicKey) -> set[Attestation]: """ Get all known attestations (made by others) for a certain public key. """ @@ -101,7 +101,7 @@ def get_attestations_for(self, public_key: PublicKey) -> typing.Set[Attestation] (public_key.key_to_bin(),), fetch_all=True)) return {Attestation.from_database_tuple(*attestation) for attestation in attestations} - def get_attestations_by(self, public_key: PublicKey) -> typing.Set[Attestation]: + def get_attestations_by(self, public_key: PublicKey) -> set[Attestation]: """ Get all attestations made by a certain public key (for others). @@ -112,7 +112,7 @@ def get_attestations_by(self, public_key: PublicKey) -> typing.Set[Attestation]: (public_key.key_to_bin(),), fetch_all=True)) return {Attestation.from_database_tuple(*attestation) for attestation in attestations} - def get_attestations_over(self, metadata: Metadata) -> typing.Set[Attestation]: + def get_attestations_over(self, metadata: Metadata) -> set[Attestation]: """ Get all known attestations for given metadata. """ @@ -147,7 +147,7 @@ def get_known_identities(self) -> list[bytes]: List the public keys of all known identity owners. """ # These are single item tuples - return [result[0] for result in typing.cast(typing.Iterator[typing.List[bytes]], + return [result[0] for result in typing.cast(typing.Iterator[list[bytes]], self.execute("SELECT public_key FROM Tokens", fetch_all=True))] def get_schema(self, version: int) -> str: diff --git a/ipv8/attestation/identity/manager.py b/ipv8/attestation/identity/manager.py index 93004c7db..da8f06639 100644 --- a/ipv8/attestation/identity/manager.py +++ b/ipv8/attestation/identity/manager.py @@ -50,7 +50,7 @@ def public_key(self) -> PublicKey: return self.tree.public_key def add_credential(self, token: Token, metadata: Metadata, - attestations: typing.Set[tuple[PublicKey, Attestation]] | None = None) -> Credential | None: + attestations: set[tuple[PublicKey, Attestation]] | None = None) -> Credential | None: """ Add a credential to this pseudonym. @@ -134,7 +134,7 @@ def get_credentials(self) -> list[Credential]: def disclose_credentials(self, credentials: typing.Iterable[Credential], - attestation_selector: typing.Set[bytes]) -> tuple[bytes, bytes, bytes, bytes]: + attestation_selector: set[bytes]) -> tuple[bytes, bytes, bytes, bytes]: """ Create a public disclosure for the given credentials and only include attestations from the given serialized public keys. @@ -148,8 +148,8 @@ def disclose_credentials(self, return self.create_disclosure({credential.metadata for credential in credentials}, attestation_selector) def create_disclosure(self, - metadata: typing.Set[Metadata], - attestation_selector: typing.Set[bytes]) -> tuple[bytes, bytes, bytes, bytes]: + metadata: set[Metadata], + attestation_selector: set[bytes]) -> tuple[bytes, bytes, bytes, bytes]: """ Create a public disclosure for the given set of metadata and only include attestations from the given serialized public keys. diff --git a/ipv8/attestation/tokentree/tree.py b/ipv8/attestation/tokentree/tree.py index 2e88bb255..401432f7b 100644 --- a/ipv8/attestation/tokentree/tree.py +++ b/ipv8/attestation/tokentree/tree.py @@ -3,7 +3,7 @@ import logging from collections import OrderedDict from hashlib import sha3_256 -from typing import TYPE_CHECKING, Set +from typing import TYPE_CHECKING from .token import Token @@ -104,7 +104,7 @@ def gather_token(self, token: Token) -> Token | None: return token return None - def get_missing(self) -> Set[bytes]: + def get_missing(self) -> set[bytes]: """ Gather all the preceding hashes that have been specified but not collected. diff --git a/ipv8/attestation/wallet/caches.py b/ipv8/attestation/wallet/caches.py index c4bf821bd..29217e50e 100644 --- a/ipv8/attestation/wallet/caches.py +++ b/ipv8/attestation/wallet/caches.py @@ -2,7 +2,7 @@ import logging from struct import unpack -from typing import TYPE_CHECKING, Any, Callable, Set +from typing import TYPE_CHECKING, Any, Callable from ...requestcache import NumberCache, RequestCache @@ -77,7 +77,7 @@ def __init__(self, community: AttestationCommunity, cache_hash: bytes, id_format Create a new cache for a pending attestation transfer. """ super().__init__(community.request_cache, "receive-verify-attestation", cache_hash, id_format) - self.attestation_map: Set[tuple[int, bytes]] = set() + self.attestation_map: set[tuple[int, bytes]] = set() def on_timeout(self) -> None: """ @@ -105,7 +105,7 @@ def __init__(self, community: AttestationCommunity, mid: bytes, key: Any, # noq Create a new cache for a pending attestation transfer reception. """ super().__init__(community.request_cache, "receive-request-attestation", mid, id_format) - self.attestation_map: Set[tuple[int, bytes]] = set() + self.attestation_map: set[tuple[int, bytes]] = set() self.key = key self.name = name diff --git a/ipv8/attestation/wallet/community.py b/ipv8/attestation/wallet/community.py index 1d01dca56..252b63551 100644 --- a/ipv8/attestation/wallet/community.py +++ b/ipv8/attestation/wallet/community.py @@ -253,7 +253,7 @@ async def on_request_attestation(self, peer: Peer, dist: GlobalTimeDistributionP self.send_attestation(peer.address, attestation_blob, dist.global_time) - def on_attestation_complete(self, unserialized: Attestation, secret_key: SecretKeyProtocol, # noqa: PLR0913 + def on_attestation_complete(self, unserialized: Attestation, secret_key: SecretKeyProtocol, peer: Peer, name: str, attestation_hash: bytes, id_format: str) -> None: """ We got an Attestation delivered to us. @@ -333,8 +333,7 @@ def send_attestation(self, socket_address: Address, blob: bytes, global_time: in If we want to serve this request, send the attestation in chunks of 800 bytes. """ - sequence_number = 0 - for i in range(0, len(blob), 800): + for sequence_number, i in enumerate(range(0, len(blob), 800)): blob_chunk = blob[i:i + 800] self.logger.debug("Sending attestation chunk %d to %s", sequence_number, str(socket_address)) if global_time is None: @@ -345,8 +344,6 @@ def send_attestation(self, socket_address: Address, blob: bytes, global_time: in packet = self._ez_pack(self._prefix, 2, [auth, dist, payload]) self.endpoint.send(socket_address, packet) - sequence_number += 1 - @lazy_wrapper(GlobalTimeDistributionPayload, AttestationChunkPayload) def on_attestation_chunk(self, peer: Peer, dist: GlobalTimeDistributionPayload, payload: AttestationChunkPayload) -> None: diff --git a/ipv8/attestation/wallet/database.py b/ipv8/attestation/wallet/database.py index 8a50e13de..3d44fb54f 100644 --- a/ipv8/attestation/wallet/database.py +++ b/ipv8/attestation/wallet/database.py @@ -1,7 +1,8 @@ from __future__ import annotations import os -from typing import TYPE_CHECKING, Any, Iterator, List, Sequence, cast +from collections.abc import Iterator, Sequence +from typing import TYPE_CHECKING, Any, cast from typing_extensions import Protocol @@ -48,7 +49,7 @@ def __init__(self, working_directory: str, db_name: str) -> None: :param db_name: The name of the database. """ if working_directory != ":memory:": - db_path = os.path.join(working_directory, os.path.join(DATABASE_DIRECTORY, "%s.db" % db_name)) + db_path = os.path.join(working_directory, os.path.join(DATABASE_DIRECTORY, f"{db_name}.db")) else: db_path = working_directory super().__init__(db_path) @@ -62,14 +63,14 @@ def get_attestation_by_hash(self, attestation_hash: bytes) -> list[bytes]: """ Retrieve a serialized attestation by hash. """ - return self._get("SELECT blob FROM %s WHERE hash = ?" % self.db_name, # noqa: S608 + return self._get(f"SELECT blob FROM {self.db_name} WHERE hash = ?", # noqa: S608 (attestation_hash,)) def get_all(self) -> list[Sequence[bytes]]: """ Get all serialized attestations we know of. """ - return list(cast(List[Sequence[bytes]], self.execute("SELECT * FROM %s" % self.db_name, # noqa: S608 + return list(cast(list[Sequence[bytes]], self.execute(f"SELECT * FROM {self.db_name}", # noqa: S608 (), fetch_all=True))) def insert_attestation(self, attestation: Attestation, attestation_hash: bytes, secret_key: SecretKeyProtocol, @@ -79,7 +80,7 @@ def insert_attestation(self, attestation: Attestation, attestation_hash: bytes, """ blob = attestation.serialize_private(secret_key.public_key()) self.execute( - "INSERT INTO %s (hash, blob, key, id_format) VALUES(?,?,?,?)" % self.db_name, + f"INSERT INTO {self.db_name} (hash, blob, key, id_format) VALUES(?,?,?,?)", (attestation_hash, blob, secret_key.serialize(), id_format.encode('utf-8'))) self.commit() @@ -90,18 +91,18 @@ def get_schema(self, version: int) -> str: """ schema = "" if version == 1: - schema = """ - CREATE TABLE IF NOT EXISTS %s( + schema = f""" + CREATE TABLE IF NOT EXISTS {self.db_name}( hash BLOB, blob LONGBLOB, key MEDIUMBLOB PRIMARY KEY (hash) ); - """ % self.db_name + """ elif version == 2: - schema = """ - CREATE TABLE IF NOT EXISTS %s( + schema = f""" + CREATE TABLE IF NOT EXISTS {self.db_name}( hash BLOB, blob LONGBLOB, key MEDIUMBLOB, @@ -109,7 +110,7 @@ def get_schema(self, version: int) -> str: PRIMARY KEY (hash) ); - """ % self.db_name + """ schema += ("CREATE TABLE IF NOT EXISTS option(key TEXT PRIMARY KEY, value BLOB);\n" "DELETE FROM option WHERE key = 'database_version';\n" f"INSERT INTO option(key, value) VALUES('database_version', '{self.LATEST_DB_VERSION!s}');\n") diff --git a/ipv8/attestation/wallet/pengbaorange/boudot.py b/ipv8/attestation/wallet/pengbaorange/boudot.py index 009de7678..3eaa9c7e3 100644 --- a/ipv8/attestation/wallet/pengbaorange/boudot.py +++ b/ipv8/attestation/wallet/pengbaorange/boudot.py @@ -9,22 +9,25 @@ from __future__ import annotations from binascii import hexlify -from math import ceil, log +from math import ceil, log2 from os import urandom from struct import pack, unpack -from typing import Iterator +from typing import TYPE_CHECKING from ..primitives.attestation import sha256_as_int from ..primitives.structs import ipack, iunpack from ..primitives.value import FP2Value +if TYPE_CHECKING: + from collections.abc import Iterator + def secure_randint(nmin: int, nmax: int) -> int: """ Generate a secure random integer. """ normalized_range = nmax - nmin - n = int(ceil(log(normalized_range, 2) / 8.0)) + n = int(ceil(log2(normalized_range) / 8.0)) rbytes_int = int(hexlify(urandom(n)), 16) return nmin + (rbytes_int % normalized_range) @@ -101,7 +104,7 @@ def create(cls: type[EL], x: int, r1: int, r2: int, # noqa: PLR0913 D2 = n2 + c * r2 return cls(c, D, D1, D2) - def check(self, # noqa: PLR0913 + def check(self, g1: FP2Value, h1: FP2Value, g2: FP2Value, @@ -167,7 +170,7 @@ def __init__(self, F: FP2Value, el: EL) -> None: # noqa: N803 self.el = el @classmethod - def create(cls: type[SQR], # noqa: PLR0913 + def create(cls: type[SQR], x: int, r1: int, g: FP2Value, diff --git a/ipv8/attestation/wallet/pengbaorange/structs.py b/ipv8/attestation/wallet/pengbaorange/structs.py index 8959f319f..44a2be6c7 100644 --- a/ipv8/attestation/wallet/pengbaorange/structs.py +++ b/ipv8/attestation/wallet/pengbaorange/structs.py @@ -87,7 +87,7 @@ class PengBaoCommitmentPrivate: MSGSPACE: list[int] = list(range(256)) - def __init__(self, m1: int, m2: int, m3: int, r1: int, r2: int, r3: int) -> None: # noqa: PLR0913 + def __init__(self, m1: int, m2: int, m3: int, r1: int, r2: int, r3: int) -> None: """ Private values for to answer challenges for our public range proof. """ @@ -173,7 +173,7 @@ class PengBaoPublicData: Public data required to verify a Peng Bao proof. """ - def __init__(self, # noqa: PLR0913 + def __init__(self, PK: BonehPublicKey, # noqa: N803 bitspace: int, commitment: PengBaoCommitment, diff --git a/ipv8/attestation/wallet/primitives/cryptography_wrapper.py b/ipv8/attestation/wallet/primitives/cryptography_wrapper.py index a11a8ec6f..93d5aa8a6 100644 --- a/ipv8/attestation/wallet/primitives/cryptography_wrapper.py +++ b/ipv8/attestation/wallet/primitives/cryptography_wrapper.py @@ -48,7 +48,6 @@ def is_prime(number: int, backend: Backend = default_backend()) -> bool: # noqa hex_n = hex(number)[2:] if hex_n.endswith('L'): hex_n = hex_n[:-1] - # hex() outputs a unicode string in Python 3 bhex_n = hex_n.encode() generated = backend._lib.BN_new() bn_pp = backend._ffi.new("BIGNUM **", generated) diff --git a/ipv8/attestation/wallet/primitives/ec.py b/ipv8/attestation/wallet/primitives/ec.py index a815cdae9..f5c632905 100644 --- a/ipv8/attestation/wallet/primitives/ec.py +++ b/ipv8/attestation/wallet/primitives/ec.py @@ -3,7 +3,7 @@ """ from __future__ import annotations -from typing import Tuple, cast +from typing import cast from .value import FP2Value @@ -21,8 +21,8 @@ def esum(mod: int, p: str | tuple[FP2Value, FP2Value], return q if q == "O": return p - x1, y1 = cast(Tuple[FP2Value, FP2Value], p) - x2, y2 = cast(Tuple[FP2Value, FP2Value], q) + x1, y1 = cast(tuple[FP2Value, FP2Value], p) + x2, y2 = cast(tuple[FP2Value, FP2Value], q) if x1 == x2 and y1 == FP2Value(mod, -1) * y2: return "O" if x1 == x2: @@ -38,8 +38,8 @@ def H(mod: int, p: str | tuple[FP2Value, FP2Value], q: str | tuple[FP2Value, FP2 """ Perform the h_{T,T} function for the Miller calculation with divisors P and Q for coordinate (x,y). """ - x1, y1 = cast(Tuple[FP2Value, FP2Value], p) - x2, y2 = cast(Tuple[FP2Value, FP2Value], q) + x1, y1 = cast(tuple[FP2Value, FP2Value], p) + x2, y2 = cast(tuple[FP2Value, FP2Value], q) if x1 == x2 and y1 == FP2Value(mod, -1) * y2: return (x - x1).normalize() if x1 == x2 and y1 == y2: @@ -71,9 +71,9 @@ def weilpairing(mod: int, m: int, P: tuple[FP2Value, FP2Value], Q: tuple[FP2Valu Create a Weil pairing for message m, points P and Q and DH secret S. """ nS = (S[0], FP2Value(mod, -1) * S[1]) - A = millercalc(mod, m, P, cast(Tuple[FP2Value, FP2Value], esum(mod, Q, S))) + A = millercalc(mod, m, P, cast(tuple[FP2Value, FP2Value], esum(mod, Q, S))) B = millercalc(mod, m, P, S) - C = millercalc(mod, m, Q, cast(Tuple[FP2Value, FP2Value], esum(mod, P, nS))) + C = millercalc(mod, m, Q, cast(tuple[FP2Value, FP2Value], esum(mod, P, nS))) D = millercalc(mod, m, Q, nS) wp = ((A * D) // (B * C)) return wp.wp_nominator() * wp.wp_denom_inverse() diff --git a/ipv8/bootstrapping/bootstrapper_interface.py b/ipv8/bootstrapping/bootstrapper_interface.py index 8c3a7183e..238cdb642 100644 --- a/ipv8/bootstrapping/bootstrapper_interface.py +++ b/ipv8/bootstrapping/bootstrapper_interface.py @@ -1,10 +1,11 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Coroutine, Iterable +from typing import TYPE_CHECKING if TYPE_CHECKING: from asyncio import Future + from collections.abc import Coroutine, Iterable from ..types import Address, Community diff --git a/ipv8/bootstrapping/dispersy/bootstrapper.py b/ipv8/bootstrapping/dispersy/bootstrapper.py index 4de2fbf52..0c1b9d3b2 100644 --- a/ipv8/bootstrapping/dispersy/bootstrapper.py +++ b/ipv8/bootstrapping/dispersy/bootstrapper.py @@ -5,12 +5,14 @@ from random import choice from socket import gethostbyname from time import time -from typing import TYPE_CHECKING, Iterable +from typing import TYPE_CHECKING from ...messaging.interfaces.udp.endpoint import UDPv4Address from ..bootstrapper_interface import Bootstrapper if TYPE_CHECKING: + from collections.abc import Iterable + from ...community import Community from ...types import Address diff --git a/ipv8/bootstrapping/udpbroadcast/bootstrapper.py b/ipv8/bootstrapping/udpbroadcast/bootstrapper.py index 5454471ea..e51062ee5 100644 --- a/ipv8/bootstrapping/udpbroadcast/bootstrapper.py +++ b/ipv8/bootstrapping/udpbroadcast/bootstrapper.py @@ -5,11 +5,13 @@ from binascii import hexlify from socket import AF_INET, SO_BROADCAST, SO_REUSEADDR, SOCK_DGRAM, SOL_SOCKET, socket from time import time -from typing import TYPE_CHECKING, Iterable +from typing import TYPE_CHECKING from ..bootstrapper_interface import Bootstrapper if TYPE_CHECKING: + from collections.abc import Iterable + from ...types import Address, Community PROTOCOL_VERSION = b'\x00\x00' @@ -34,7 +36,7 @@ def __init__(self, overlay: Community) -> None: self.overlay = overlay self.logger = logging.getLogger(self.__class__.__name__) - async def open(self) -> bool: # noqa: A003 + async def open(self) -> bool: """ Open the broadcast socket. """ diff --git a/ipv8/community.py b/ipv8/community.py index 4c0dc507e..f69d1cf74 100644 --- a/ipv8/community.py +++ b/ipv8/community.py @@ -12,11 +12,12 @@ import sys from asyncio import ensure_future, iscoroutine from binascii import hexlify +from collections.abc import Awaitable from itertools import islice from random import choice, random from time import time from traceback import format_exception -from typing import TYPE_CHECKING, Awaitable, Callable, cast +from typing import TYPE_CHECKING, Callable, cast from .lazy_community import EZPackOverlay, lazy_wrapper, lazy_wrapper_unsigned from .messaging.anonymization.endpoint import TunnelEndpoint diff --git a/ipv8/configuration.py b/ipv8/configuration.py index 5afbe8a32..8c2dd3f39 100644 --- a/ipv8/configuration.py +++ b/ipv8/configuration.py @@ -168,7 +168,7 @@ def values(cls: type[Strategy]) -> list[str]: """ Get the known default DiscoveryStrategy names. """ - return list(typing.cast(typing.Dict[str, Any], cls.__members__)) + return list(typing.cast(dict[str, Any], cls.__members__)) class WalkerDefinition(typing.NamedTuple): @@ -197,7 +197,7 @@ def values(cls: type[Bootstrapper]) -> list[str]: """ Get the known default Bootstrapper names. """ - return list(typing.cast(typing.Dict[str, Any], cls.__members__)) + return list(typing.cast(dict[str, Any], cls.__members__)) class BootstrapperDefinition(typing.NamedTuple): diff --git a/ipv8/database.py b/ipv8/database.py index 94bb07e1b..4ee1c8a0b 100644 --- a/ipv8/database.py +++ b/ipv8/database.py @@ -12,9 +12,10 @@ import sqlite3 from abc import ABCMeta, abstractmethod from collections import defaultdict +from collections.abc import Iterable, Iterator, Mapping from sqlite3 import Connection, Cursor, OperationalError from threading import RLock -from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, Mapping, Tuple, Union, cast +from typing import TYPE_CHECKING, Any, Callable, Union, cast if TYPE_CHECKING: from types import TracebackType @@ -82,7 +83,7 @@ def __init__(self, file_path: str) -> None: @type file_path: unicode """ self._assert(isinstance(file_path, str), - "expected file_path to be unicode, but was %s" % str(type(file_path))) + f"expected file_path to be unicode, but was {type(file_path)!s}") super().__init__() self._logger = logging.getLogger(self.__class__.__name__) @@ -106,7 +107,7 @@ def _assert(self, condition: bool, message: str = "") -> None: if not condition: raise DatabaseException(str(message)) - def open(self, initial_statements: bool = True, prepare_visioning: bool = True) -> bool: # noqa: A003 + def open(self, initial_statements: bool = True, prepare_visioning: bool = True) -> bool: """ Open a connection to the underlying database file. """ @@ -221,7 +222,7 @@ def _prepare_version(self) -> None: if count: # get version from required 'option' table try: - version, = next(cast(Iterator[Tuple[bytes]], self.execute("SELECT value FROM option " + version, = next(cast(Iterator[tuple[bytes]], self.execute("SELECT value FROM option " "WHERE key == 'database_version' " "LIMIT 1"))) except OperationalError: @@ -233,7 +234,7 @@ def _prepare_version(self) -> None: self._database_version = self.check_database(version) self._assert(isinstance(self._database_version, int), - "expected databse version to be int or long, but was type %s" % str(type(self._database_version))) + f"expected databse version to be int or long, but was type {type(self._database_version)!s}") @property def database_version(self) -> int: diff --git a/ipv8/dht/churn.py b/ipv8/dht/churn.py index 071a59feb..3c626cf5c 100644 --- a/ipv8/dht/churn.py +++ b/ipv8/dht/churn.py @@ -19,7 +19,7 @@ def __init__(self, overlay: DHTCommunity, ping_interval: float = 25.0) -> None: super().__init__(overlay) self.ping_interval = ping_interval - def take_step(self) -> None: # noqa: C901 + def take_step(self) -> None: # noqa: C901, PLR0912 """ Every tick (half-second by default), performs maintainence. diff --git a/ipv8/dht/community.py b/ipv8/dht/community.py index 69aced349..2f4ab9731 100644 --- a/ipv8/dht/community.py +++ b/ipv8/dht/community.py @@ -8,8 +8,9 @@ from asyncio import FIRST_COMPLETED, Future, Task, gather, wait from binascii import hexlify, unhexlify from collections import defaultdict, deque +from collections.abc import Coroutine, Iterator, Sequence from itertools import zip_longest -from typing import TYPE_CHECKING, Any, Coroutine, Iterator, List, Optional, Sequence, Set, Tuple, cast +from typing import TYPE_CHECKING, Any, Optional, cast from ..community import Community, CommunitySettings from ..lazy_community import lazy_wrapper, lazy_wrapper_wd @@ -46,7 +47,7 @@ from ..peerdiscovery.discovery import DiscoveryStrategy from ..types import Address, Peer -DHTValue = Tuple[bytes, Optional[bytes]] +DHTValue = tuple[bytes, Optional[bytes]] PING_INTERVAL = 25 @@ -87,7 +88,7 @@ class Request(RandomNumberCache): This request cache keeps track of all outstanding requests within the DHTCommunity. """ - def __init__(self, community: DHTCommunity, msg_type: str, node: Node, # noqa: PLR0913 + def __init__(self, community: DHTCommunity, msg_type: str, node: Node, params: list | None = None, consume_errors: bool = False, timeout: float = 5.0) -> None: """ Create a new request state. @@ -164,7 +165,7 @@ def __init__(self, target: bytes, routing_table: RoutingTable, force_nodes: bool # Keep a list of nodes that still need to be contacted: [(node_to_contact, node_to_puncture)] self.nodes_todo: list[Crawl.TodoNode] = [Crawl.TodoNode(n, None) for n in nodes_closest] - self.nodes_tried: Set[Node] = set() + self.nodes_tried: set[Node] = set() self.responses: list[tuple[Node, dict[str, list[bytes] | list[Node]]]] = [] self.force_nodes = force_nodes @@ -176,7 +177,7 @@ def add_response(self, sender: Node, response: dict[str, list[bytes] | list[Node """ self.responses.append((sender, response)) - for index, node in enumerate(cast(List[Node], response.get('nodes', []))): + for index, node in enumerate(cast(list[Node], response.get('nodes', []))): if node in self.nodes_tried: continue @@ -215,7 +216,7 @@ def values(self) -> list[bytes]: Get the currently known values for our crawl. """ # Merge all values received into one tuple. First pick the first value from each tuple, then the second, etc. - value_responses: list[list[bytes]] = [cast(List[bytes], response['values']) for _, response in self.responses + value_responses: list[list[bytes]] = [cast(list[bytes], response['values']) for _, response in self.responses if 'values' in response] values = sum(zip_longest(*value_responses), ()) @@ -296,7 +297,7 @@ def get_address_class(self, node: Peer) -> type[Address]: """ Get the class of the given node's address. """ - return node.address.__class__ if node.address.__class__ != tuple else UDPv4Address + return node.address.__class__ if isinstance(node.address, tuple) else UDPv4Address def get_routing_table(self, node: Peer) -> RoutingTable: """ @@ -466,8 +467,8 @@ async def _store(self, key: bytes, value: bytes) -> list[Node]: msg = "Maximum length exceeded" raise DHTError(msg) - nodes = cast(List[Node], await self.find_nodes(key)) - nodes = cast(List[Node], await self.store_on_nodes(key, [value], nodes[:TARGET_NODES])) + nodes = cast(list[Node], await self.find_nodes(key)) + nodes = cast(list[Node], await self.store_on_nodes(key, [value], nodes[:TARGET_NODES])) if len(nodes) < 1: msg = "Failed to store value on DHT" raise DHTError(msg) @@ -575,7 +576,7 @@ async def _contact_node(self, crawl: Crawl, node: Node, puncture_node: Node) -> async def _find(self, crawl: Crawl, debug: bool = False) -> list[Node] | list[DHTValue] | \ tuple[list[DHTValue], Crawl]: - tasks: Set[Future | Task] = set() + tasks: set[Future | Task] = set() while True: # Keep running tasks until work is done. while not crawl.done and len(tasks) < MAX_CRAWL_TASKS: @@ -641,7 +642,7 @@ async def find(self, target: bytes, force_nodes: bool, offset: int, tuple[list[DHTValue], Crawl]] = await gather(*futures) if debug: - results_debug = cast(List[Tuple[List[DHTValue], Crawl]], results) + results_debug = cast(list[tuple[list[DHTValue], Crawl]], results) return tuple(*[r[0] for r in results]), [r[1] for r in results_debug] return tuple(*results) @@ -651,8 +652,8 @@ async def find_values(self, target: bytes, offset: int = 0, Find the values belonging to the target key. """ values = await self.find(target, False, offset, debug) - return (cast(Tuple[Sequence[Tuple[bytes, Optional[bytes]]], List[Crawl]], values) if debug - else cast(Sequence[Tuple[bytes, Optional[bytes]]], values)) + return (cast(tuple[Sequence[tuple[bytes, Optional[bytes]]], list[Crawl]], values) if debug + else cast(Sequence[tuple[bytes, Optional[bytes]]], values)) async def find_nodes(self, target: bytes, debug: bool = False) -> Sequence[Node]: """ @@ -705,7 +706,7 @@ def on_find_response(self, peer: Peer, payload: FindResponsePayload) -> None: # The errback must already have been called (due to a timeout) return - if cast(List[bool], cache.params)[0]: + if cast(list[bool], cache.params)[0]: cache.future.set_result({'nodes': payload.nodes}) else: cache.future.set_result({'values': payload.values} if payload.values diff --git a/ipv8/dht/discovery.py b/ipv8/dht/discovery.py index 4dd2e8170..9044e4f98 100644 --- a/ipv8/dht/discovery.py +++ b/ipv8/dht/discovery.py @@ -1,9 +1,11 @@ from __future__ import annotations +import functools +import operator import time from binascii import hexlify from collections import defaultdict -from typing import TYPE_CHECKING, Callable, List, cast +from typing import TYPE_CHECKING, Callable, cast from ..lazy_community import lazy_wrapper, lazy_wrapper_wd from ..types import Address @@ -87,7 +89,7 @@ async def store_peer(self) -> list[Node]: return [] try: - nodes = cast(List[Node], await self.find_nodes(key)) + nodes = cast(list[Node], await self.find_nodes(key)) return await self.send_store_peer_request(key, nodes[:TARGET_NODES]) except DHTError: return [] @@ -136,7 +138,7 @@ async def connect_peer(self, mid: bytes, peer: Peer | None = None) -> list[Node] else: return [node] - nodes = cast(List[Node], await self.find_nodes(mid)) + nodes = cast(list[Node], await self.find_nodes(mid)) nodes = await self.send_connect_peer_request(mid, nodes[:TARGET_NODES]) if not nodes: msg = "Failed to connect peer" @@ -162,7 +164,7 @@ async def send_connect_peer_request(self, key: bytes, nodes: list[Node]) -> list self.ez_send(node, ConnectPeerRequestPayload(cache.number, self.my_estimated_lan, key)) node_lists = await gather_without_errors(*futures) - return list(set(sum(node_lists, []))) + return list(set(functools.reduce(operator.iadd, node_lists, []))) @lazy_wrapper(StorePeerRequestPayload) def on_store_peer_request(self, peer: Peer, payload: StorePeerRequestPayload) -> None: @@ -200,7 +202,7 @@ def on_store_peer_response(self, peer: Peer, payload: StorePeerResponsePayload) cache = cast(Request, self.request_cache.pop('store-peer', payload.identifier)) - key = cast(List[bytes], cache.params)[0] + key = cast(list[bytes], cache.params)[0] if cache.node not in self.store_for_me[key]: self.logger.debug('Peer %s storing us (key %s)', cache.node, hexlify(key)) self.store_for_me[key].append(cache.node) diff --git a/ipv8/dht/provider.py b/ipv8/dht/provider.py index 7e9abf61a..e34963761 100644 --- a/ipv8/dht/provider.py +++ b/ipv8/dht/provider.py @@ -2,7 +2,7 @@ import logging from binascii import hexlify -from typing import TYPE_CHECKING, List, Tuple, cast +from typing import TYPE_CHECKING, cast from ..messaging.anonymization.tunnel import PEER_SOURCE_DHT, IntroductionPoint from ..messaging.lazy_payload import VariablePayload, vp_compile @@ -67,7 +67,7 @@ async def lookup(self, info_hash: bytes) -> tuple[bytes, list[IntroductionPoint] return None results = [] - for value, _ in cast(Tuple[List[bytes], List[bytes]], values): + for value, _ in cast(tuple[list[bytes], list[bytes]], values): try: payload, _ = default_serializer.unpack_serializable(DHTIntroPointPayload, value) intro_peer = Peer(b'LibNaCLPK:' + payload.intro_pk, payload.address) diff --git a/ipv8/dht/routing.py b/ipv8/dht/routing.py index f8bd3d586..19b47e858 100644 --- a/ipv8/dht/routing.py +++ b/ipv8/dht/routing.py @@ -56,7 +56,7 @@ def calc_node_id(address: Address | UDPv4Address | UDPv6Address, mid: bytes) -> ip_masked = bytes([ip_bin[i] & ip_mask[i] for i in range(4)]) crc32_unsigned = binascii.crc32(ip_masked) % (2 ** 32) - crc32_bin = binascii.unhexlify('%08x' % crc32_unsigned) + crc32_bin = binascii.unhexlify(f'{crc32_unsigned:08x}') return crc32_bin[:3] + mid[:17] @@ -79,7 +79,7 @@ def __init__(self, key: Key | bytes, address: Address | None = None, intro: bool self.rtt: float = 0 @property - def id(self) -> bytes: # noqa: A003 + def id(self) -> bytes: """ The id of this node. """ diff --git a/ipv8/dht/trie.py b/ipv8/dht/trie.py index c2d38deaf..b7a5ea4e7 100644 --- a/ipv8/dht/trie.py +++ b/ipv8/dht/trie.py @@ -1,9 +1,12 @@ from __future__ import annotations -from typing import Generic, Iterator, Tuple, TypeVar, cast +from typing import TYPE_CHECKING, Generic, TypeVar, cast from . import DHTError +if TYPE_CHECKING: + from collections.abc import Iterator + # Sentinel object NullType = object Null = NullType() @@ -132,7 +135,7 @@ def longest_prefix_item(self, key: str, default: tuple[str, ValueType] | NullTyp if value: return prefix, value if default is not Null: - return cast(Tuple[str, ValueType], default) + return cast(tuple[str, ValueType], default) raise KeyError def longest_prefix(self, key: str, default: str | NullType = Null) -> str: diff --git a/ipv8/keyvault/crypto.py b/ipv8/keyvault/crypto.py index 23dbd2021..0343e438b 100644 --- a/ipv8/keyvault/crypto.py +++ b/ipv8/keyvault/crypto.py @@ -72,7 +72,8 @@ def generate_key(self, security_level: str) -> PrivateKey: if curve[1] == "libnacl": return LibNaCLSK() - raise RuntimeError("Illegal curve for key generation: %s" % security_level) + msg = f"Illegal curve for key generation: {security_level}" + raise RuntimeError(msg) def key_to_bin(self, ec: Key) -> bytes: """ diff --git a/ipv8/lazy_community.py b/ipv8/lazy_community.py index c014f917f..6dc54a273 100644 --- a/ipv8/lazy_community.py +++ b/ipv8/lazy_community.py @@ -2,7 +2,7 @@ from abc import ABC, abstractmethod from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Coroutine, Sequence, Tuple, TypeVar, cast +from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast from .keyvault.crypto import default_eccrypto from .messaging.payload_headers import BinMemberAuthenticationPayload, GlobalTimeDistributionPayload @@ -22,6 +22,8 @@ ) if TYPE_CHECKING: + from collections.abc import Coroutine, Sequence + from .messaging.serialization import Serializable UT = TypeVar("UT", bound=Payload) @@ -102,8 +104,9 @@ def wrapper(self: EZPackOverlay, source_address: Address, data: bytes) -> Corout unpacked = self.serializer.unpack_serializable_list(payloads, remainder, offset=23) # ASSERT if not signature_valid: - raise PacketDecodingError("Incoming packet %s has an invalid signature" % - str([payload_class.__name__ for payload_class in payloads])) + msg = (f"Incoming packet {[payload_class.__name__ for payload_class in payloads]!s}" + " has an invalid signature") + raise PacketDecodingError(msg) # PRODUCE peer = self.network.verified_by_public_key_bin.get(auth.public_key_bin) if peer: @@ -138,8 +141,8 @@ def wrapper(self: EZPackOverlay, source_address: Address, data: bytes) -> Corout unpacked = self.serializer.unpack_serializable_list(payloads, remainder, offset=23) # ASSERT if not signature_valid: - raise PacketDecodingError("Incoming packet %s has an invalid signature" % - str([payload_class.__name__ for payload_class in payloads])) + msg = f"Incoming packet {[payload_class.__name__ for payload_class in payloads]!s} has an invalid signature" + raise PacketDecodingError(msg) # PRODUCE output = [*unpacked, data] peer = self.network.verified_by_public_key_bin.get(auth.public_key_bin) @@ -284,7 +287,8 @@ def _ez_unpack_auth(self, unpacked = self.serializer.unpack_serializable_list(format, remainder, offset=23) # ASSERT if not signature_valid: - raise PacketDecodingError("Incoming packet %s has an invalid signature" % payload_class.__name__) + msg = f"Incoming packet {payload_class.__name__} has an invalid signature" + raise PacketDecodingError(msg) # PRODUCE return auth, cast(GlobalTimeDistributionPayload, unpacked[0]), cast(UT, unpacked[1]) @@ -297,7 +301,7 @@ def _ez_unpack_noauth(self, else [payload_class]) unpacked = self.serializer.unpack_serializable_list(format, data, offset=23) # PRODUCE - return (cast(Tuple[GlobalTimeDistributionPayload, UT], unpacked) if global_time + return (cast(tuple[GlobalTimeDistributionPayload, UT], unpacked) if global_time else cast(UT, unpacked[0])) diff --git a/ipv8/loader.py b/ipv8/loader.py index c03b6452b..c4edeca6c 100644 --- a/ipv8/loader.py +++ b/ipv8/loader.py @@ -8,7 +8,7 @@ import abc import logging import types -from typing import TYPE_CHECKING, Any, Callable, Type, cast +from typing import TYPE_CHECKING, Any, Callable, cast from .keyvault.crypto import default_eccrypto from .peer import Peer @@ -176,7 +176,8 @@ def load(self, overlay_provider: IPv8, session: object) -> None: remaining.append(launcher) if cycle < 0: launcher_names = [launcher.get_name() for launcher in remaining] - raise RuntimeError("Cycle detected in CommunityLauncher not_before(): %s" % (str(launcher_names))) + msg = f"Cycle detected in CommunityLauncher not_before(): {launcher_names!s}" + raise RuntimeError(msg) self._logger.info("Finished loading communities!") @abc.abstractmethod @@ -300,7 +301,7 @@ def decorator(instance: type[CommunityLauncher]) -> type[CommunityLauncher]: def new_should_launch(self: CommunityLauncher, session: object) -> bool: return (old_should_launch(self, session) - and eval(str_condition, globals(), locals())) # noqa: PGH001, S307 + and eval(str_condition, globals(), locals())) # noqa: S307 instance.should_launch = new_should_launch # type: ignore[method-assign] return instance @@ -314,9 +315,9 @@ def _get_class(class_or_function: type[Community] | Callable[[Any], type[Communi :param class_or_function: the class, or the function that represents this class. """ - return (cast(Type[Community], class_or_function()) # type: ignore[call-arg, method-assign] + return (cast(type[Community], class_or_function()) # type: ignore[call-arg, method-assign] if isinstance(class_or_function, types.FunctionType) - else cast(Type[Community], class_or_function)) + else cast(type[Community], class_or_function)) def overlay(str_module_or_class: str | type[Community] | Callable[[], type[Community]], @@ -360,7 +361,7 @@ def get_overlay_class(self: CommunityLauncher) -> type[Community]: cast(str, str_definition)) else: def get_overlay_class(self: CommunityLauncher) -> type[Community]: - return _get_class(cast(Type[Community], str_module_or_class)) + return _get_class(cast(type[Community], str_module_or_class)) instance.get_overlay_class = get_overlay_class # type: ignore[method-assign] return instance @@ -416,7 +417,7 @@ def decorator(instance: type[CommunityLauncher]) -> type[CommunityLauncher]: fromlist=[cast(str, str_definition)]), cast(str, str_definition)) else: - strategy_class = _get_class(cast(Type[Community], str_module_or_class)) + strategy_class = _get_class(cast(type[Community], str_module_or_class)) def new_get_walk_strategies(self: CommunityLauncher) -> list[tuple[type[DiscoveryStrategy], dict, int]]: return [*old_get_walk_strategies(self), (strategy_class, kw_args or {}, target_peers)] @@ -473,7 +474,7 @@ def decorator(instance: type[CommunityLauncher]) -> type[CommunityLauncher]: fromlist=[cast(str, str_definition)]), cast(str, str_definition)) else: - bootstrapper_class = _get_class(cast(Type[Community], str_module_or_class)) + bootstrapper_class = _get_class(cast(type[Community], str_module_or_class)) def new_get_bootstrappers(self: CommunityLauncher, session: object) -> list[tuple[type[Bootstrapper], dict]]: return [*old_get_bootstrappers(self, session), (bootstrapper_class, kw_args or {})] @@ -535,7 +536,7 @@ def decorator(instance: type[CommunityLauncher]) -> type[CommunityLauncher]: def new_get_kwargs(self: CommunityLauncher, session: object) -> dict: out = old_get_kwargs(self, session) for kwarg in kw_args: - out[kwarg] = eval(kw_args[kwarg], globals(), locals()) # noqa: PGH001, S307 + out[kwarg] = eval(kw_args[kwarg], globals(), locals()) # noqa: S307 return out instance.get_kwargs = new_get_kwargs # type: ignore[method-assign] diff --git a/ipv8/messaging/anonymization/caches.py b/ipv8/messaging/anonymization/caches.py index a57f186fb..756348e08 100644 --- a/ipv8/messaging/anonymization/caches.py +++ b/ipv8/messaging/anonymization/caches.py @@ -24,7 +24,7 @@ class CreateRequestCache(RandomNumberCacheWithName): name = "create" - def __init__(self, community: TunnelCommunity, identifier: int, to_circuit_id: int, # noqa: PLR0913 + def __init__(self, community: TunnelCommunity, identifier: int, to_circuit_id: int, from_circuit_id: int, peer: Peer, to_peer: Peer) -> None: """ Create a new cache. @@ -81,7 +81,7 @@ class RetryRequestCache(NumberCacheWithName): name = "retry" - def __init__(self, community: TunnelCommunity, circuit: Circuit, # noqa: PLR0913 + def __init__(self, community: TunnelCommunity, circuit: Circuit, candidates: list[bytes] | list[Peer], max_tries: int, retry_func: Callable[[Circuit, list[bytes], int], None] | Callable[[Circuit, list[Peer], int], None], timeout: float) -> None: diff --git a/ipv8/messaging/anonymization/community.py b/ipv8/messaging/anonymization/community.py index 4209dc35d..b95b6a7c2 100644 --- a/ipv8/messaging/anonymization/community.py +++ b/ipv8/messaging/anonymization/community.py @@ -11,8 +11,9 @@ from asyncio import ensure_future, iscoroutine, sleep from binascii import unhexlify from collections import defaultdict +from collections.abc import Awaitable from traceback import format_exception -from typing import TYPE_CHECKING, Awaitable, List, Optional, Set +from typing import TYPE_CHECKING, Optional from ...community import Community, CommunitySettings from ...keyvault.private.libnaclkey import LibNaCLSK @@ -103,7 +104,7 @@ class TunnelSettings(CommunitySettings): # to flow over the circuit (i.e. bandwidth payouts to intermediate nodes in a circuit). remove_tunnel_delay = 5 - _peer_flags: Set[int] = {PEER_FLAG_RELAY, PEER_FLAG_SPEED_TEST} + _peer_flags: set[int] = {PEER_FLAG_RELAY, PEER_FLAG_SPEED_TEST} _max_relay_early = 8 @@ -126,14 +127,14 @@ def max_relay_early(self, value: int) -> None: self.endpoint.set_max_relay_early(value) @property - def peer_flags(self) -> Set[int]: + def peer_flags(self) -> set[int]: """ Return the peer flags. """ return self._peer_flags @peer_flags.setter - def peer_flags(self, value: Set[int]) -> None: + def peer_flags(self, value: set[int]) -> None: """ Set the peer flags. """ @@ -584,7 +585,7 @@ def _ours_on_created_extended(self, circuit_id: int, payload: CreatedPayload | E candidates, _ = self.serializer.unpack('varlenH-list', candidates_bin) cache = self.request_cache.pop(RetryRequestCache, circuit.circuit_id) - self.send_extend(circuit, cast(List[bytes], candidates), cache.max_tries if cache else 1) + self.send_extend(circuit, cast(list[bytes], candidates), cache.max_tries if cache else 1) elif circuit.state == CIRCUIT_STATE_READY: self.request_cache.pop(RetryRequestCache, circuit.circuit_id) diff --git a/ipv8/messaging/anonymization/exit_socket.py b/ipv8/messaging/anonymization/exit_socket.py index 0d57b4e2d..9b34f1f90 100644 --- a/ipv8/messaging/anonymization/exit_socket.py +++ b/ipv8/messaging/anonymization/exit_socket.py @@ -52,9 +52,7 @@ def could_be_utp(data: bytes) -> bool: if not (0 <= (byte1 >> 4) <= 4 and (byte1 & 15) == 1): return False # Extension - if not (0 <= byte2 <= 3): - return False - return True + return 0 <= byte2 <= 3 @staticmethod def could_be_udp_tracker(data: bytes) -> bool: @@ -62,10 +60,7 @@ def could_be_udp_tracker(data: bytes) -> bool: Check if the data could be a UDP-based tracker. """ # For the UDP tracker protocol the action field is either at position 0 or 8, and should be 0..3 - if len(data) >= 8 and (0 <= unpack_from('!I', data, 0)[0] <= 3)\ - or len(data) >= 12 and (0 <= unpack_from('!I', data, 8)[0] <= 3): - return True - return False + return bool(len(data) >= 8 and 0 <= unpack_from('!I', data, 0)[0] <= 3 or len(data) >= 12 and 0 <= unpack_from('!I', data, 8)[0] <= 3) @staticmethod def could_be_dht(data: bytes) -> bool: @@ -109,7 +104,7 @@ def __init__(self, received_cb: Callable, local_addr: Address) -> None: self.local_addr = local_addr self.logger = logging.getLogger(self.__class__.__name__) - async def open(self) -> DatagramTransport: # noqa: A003 + async def open(self) -> DatagramTransport: """ Opens a datagram endpoint and returns the Transport. """ @@ -184,7 +179,7 @@ def on_address(future: Future[Address]) -> None: task = ensure_future(self.resolve(destination)) # If this fails, the TaskManager logs the packet. - self.register_anonymous_task("resolving_%r" % destination[0], task, + self.register_anonymous_task(f"resolving_{destination[0]!r}", task, ignore=(OSError, ValueError)).add_done_callback(on_address) return diff --git a/ipv8/messaging/anonymization/hidden_services.py b/ipv8/messaging/anonymization/hidden_services.py index 89f2a4823..59d0f5271 100644 --- a/ipv8/messaging/anonymization/hidden_services.py +++ b/ipv8/messaging/anonymization/hidden_services.py @@ -11,7 +11,7 @@ import socket import struct from asyncio import gather, iscoroutine -from typing import TYPE_CHECKING, Any, Coroutine, Set, Tuple, cast +from typing import TYPE_CHECKING, Any, cast from ...bootstrapping.dispersy.bootstrapper import DispersyBootstrapper from ...configuration import DISPERSY_BOOTSTRAPPER @@ -46,6 +46,8 @@ ) if TYPE_CHECKING: + from collections.abc import Coroutine + from ...types import IPv8 @@ -85,11 +87,11 @@ def __init__(self, settings: HiddenTunnelSettings) -> None: # Messages that can arrive from the socket # The circuit id is optional, so we can safely cast these handlers. - self.add_message_handler(CreateE2EPayload, cast(Callable[[Tuple[str, int], bytes], None], + self.add_message_handler(CreateE2EPayload, cast(Callable[[tuple[str, int], bytes], None], self.on_create_e2e)) - self.add_message_handler(PeersRequestPayload, cast(Callable[[Tuple[str, int], bytes], None], + self.add_message_handler(PeersRequestPayload, cast(Callable[[tuple[str, int], bytes], None], self.on_peers_request)) - self.add_message_handler(PeersResponsePayload, cast(Callable[[Tuple[str, int], bytes], None], + self.add_message_handler(PeersResponsePayload, cast(Callable[[tuple[str, int], bytes], None], self.on_peers_response)) # Messages that can arrive from a circuit (i.e., they are wrapped in a cell) @@ -155,7 +157,7 @@ async def estimate_swarm_size(self, info_hash: bytes, hops: int = 1, max_request # None represents a DHT request all_: list[IntroductionPoint | None] = [None] - tried: Set[IntroductionPoint | None] = set() + tried: set[IntroductionPoint | None] = set() while len(tried) < max_requests: not_tried = set(all_) - tried @@ -185,7 +187,7 @@ def select_circuit_for_infohash(self, info_hash: bytes) -> Circuit | None: return self.select_circuit(None, swarm.hops) - def create_circuit_for_infohash(self, info_hash: bytes, ctype: str, exit_flags: Set[int] | None = None, + def create_circuit_for_infohash(self, info_hash: bytes, ctype: str, exit_flags: set[int] | None = None, required_exit: Peer | None = None) -> Circuit | None: """ Create a circuit that connects to the swarm given by the SHA-1 (info) hash. @@ -498,7 +500,7 @@ async def on_created_e2e(self, source_address: Address, payload: CreatedE2EPaylo rp_info_enc = payload.rp_info_enc rp_info_bin = self.crypto.decrypt_str(rp_info_enc, session_keys, FORWARD) - rp_info, _ = cast(Tuple[RendezvousInfo, int], self.serializer.unpack(RendezvousInfo, rp_info_bin)) + rp_info, _ = cast(tuple[RendezvousInfo, int], self.serializer.unpack(RendezvousInfo, rp_info_bin)) required_exit = Peer(rp_info.key, rp_info.address) circuit = self.create_circuit_for_infohash(cache.info_hash, CIRCUIT_TYPE_RP_DOWNLOADER, diff --git a/ipv8/messaging/anonymization/pex.py b/ipv8/messaging/anonymization/pex.py index 81b5835f4..c8df03d86 100644 --- a/ipv8/messaging/anonymization/pex.py +++ b/ipv8/messaging/anonymization/pex.py @@ -3,7 +3,7 @@ import random import time from collections import deque -from typing import TYPE_CHECKING, List, cast +from typing import TYPE_CHECKING, cast from ...community import Community, CommunitySettings from ...messaging.anonymization.tunnel import PEER_SOURCE_PEX, IntroductionPoint @@ -92,7 +92,7 @@ def process_extra_bytes(self, peer: Peer, extra_bytes: bytes) -> None: if not extra_bytes: return - for seeder_pk in cast(List[bytes], self.serializer.unpack('varlenH-list', extra_bytes)[0]): + for seeder_pk in cast(list[bytes], self.serializer.unpack('varlenH-list', extra_bytes)[0]): ip = IntroductionPoint(peer, seeder_pk, PEER_SOURCE_PEX) if ip in self.intro_points: # Remove first to put introduction point at front of the deque. diff --git a/ipv8/messaging/anonymization/tunnel.py b/ipv8/messaging/anonymization/tunnel.py index ba0960de2..b5a229976 100644 --- a/ipv8/messaging/anonymization/tunnel.py +++ b/ipv8/messaging/anonymization/tunnel.py @@ -6,11 +6,13 @@ from asyncio import Future, gather from binascii import hexlify from dataclasses import dataclass -from typing import TYPE_CHECKING, Callable, Sequence, cast +from typing import TYPE_CHECKING, Callable, cast from ...keyvault.public.libnaclkey import LibNaCLPK if TYPE_CHECKING: + from collections.abc import Sequence + from ...keyvault.private.libnaclkey import LibNaCLSK from ...peer import Peer from ...types import Address diff --git a/ipv8/messaging/anonymization/utils.py b/ipv8/messaging/anonymization/utils.py index 14e0c2bab..1a66e376d 100644 --- a/ipv8/messaging/anonymization/utils.py +++ b/ipv8/messaging/anonymization/utils.py @@ -3,7 +3,7 @@ from asyncio import FIRST_COMPLETED, Future, wait from statistics import mean, median from timeit import default_timer -from typing import TYPE_CHECKING, Set +from typing import TYPE_CHECKING from .tunnel import CIRCUIT_STATE_CLOSING, Circuit @@ -18,7 +18,7 @@ async def run_speed_test(tc: TunnelCommunity, circuit: Circuit, request_size: in """ num_sent = 0 num_ack = 0 - outstanding: Set[Future[tuple[bytes, float]]] = set() + outstanding: set[Future[tuple[bytes, float]]] = set() start = default_timer() rtts = [] diff --git a/ipv8/messaging/interfaces/endpoint.py b/ipv8/messaging/interfaces/endpoint.py index ed51113f0..4ccfc876d 100644 --- a/ipv8/messaging/interfaces/endpoint.py +++ b/ipv8/messaging/interfaces/endpoint.py @@ -6,11 +6,13 @@ import socket import struct import threading -from typing import TYPE_CHECKING, Awaitable, Iterable +from typing import TYPE_CHECKING from .lan_addresses.interfaces import get_lan_addresses if TYPE_CHECKING: + from collections.abc import Awaitable, Iterable + from ...types import Address diff --git a/ipv8/messaging/interfaces/lan_addresses/addressprovider.py b/ipv8/messaging/interfaces/lan_addresses/addressprovider.py index 2ac9a5c29..947b41a55 100644 --- a/ipv8/messaging/interfaces/lan_addresses/addressprovider.py +++ b/ipv8/messaging/interfaces/lan_addresses/addressprovider.py @@ -1,7 +1,6 @@ from __future__ import annotations import traceback -import typing from abc import ABC, abstractmethod from time import time @@ -18,7 +17,7 @@ def __init__(self, verbose: bool = False) -> None: :param verbose: Log any errors that are encountered while fetching addresses. """ self.verbose = verbose - self.addresses: typing.Set[str] = set() + self.addresses: set[str] = set() self.addresses_ts = 0.0 def on_exception(self) -> None: @@ -44,12 +43,12 @@ def discover_addresses(self, min_interval: float = 10.0) -> None: self.addresses_ts = time() @abstractmethod - def get_addresses(self) -> typing.Set[str]: + def get_addresses(self) -> set[str]: """ Get a set of LAN addresses using this provider. """ - def get_addresses_buffered(self) -> typing.Set[str]: + def get_addresses_buffered(self) -> set[str]: """ Return the known addresses from when ``discover_addresses()`` was last successfully called. If discovery hasn't been performed yet, do so now. diff --git a/ipv8/messaging/interfaces/lan_addresses/unix/getifaddrs.py b/ipv8/messaging/interfaces/lan_addresses/unix/getifaddrs.py index f17cc0e8b..ab5e733bc 100644 --- a/ipv8/messaging/interfaces/lan_addresses/unix/getifaddrs.py +++ b/ipv8/messaging/interfaces/lan_addresses/unix/getifaddrs.py @@ -85,7 +85,7 @@ class ifaddrs(Structure): """ -ifaddrs._fields_ = [ # noqa: SLF001 +ifaddrs._fields_ = [ ('ifa_next', POINTER(ifaddrs)), ('ifa_name', c_char_p), ('ifa_flags', c_uint), diff --git a/ipv8/messaging/interfaces/lan_addresses/windows/GetAdaptersAddresses.py b/ipv8/messaging/interfaces/lan_addresses/windows/GetAdaptersAddresses.py index cb13271a6..0437a177d 100644 --- a/ipv8/messaging/interfaces/lan_addresses/windows/GetAdaptersAddresses.py +++ b/ipv8/messaging/interfaces/lan_addresses/windows/GetAdaptersAddresses.py @@ -78,7 +78,7 @@ def __getattr__(self, name: str) -> typing.Any: # noqa: ANN401 """ ... # incomplete - class _GetAdaptersAddresses(typing.Protocol, typing.SupportsInt): # noqa: PYI046 + class _GetAdaptersAddresses(typing.Protocol, typing.SupportsInt): def __call__(self, Family: c_ulong, Flags: c_ulong, Reserved: int, AdapterAddresses: _Pointer, SizePointer: _Pointer) -> int: ... diff --git a/ipv8/messaging/interfaces/udp/endpoint.py b/ipv8/messaging/interfaces/udp/endpoint.py index c703a0de7..ccb9990bd 100644 --- a/ipv8/messaging/interfaces/udp/endpoint.py +++ b/ipv8/messaging/interfaces/udp/endpoint.py @@ -102,7 +102,7 @@ def log_error(self, message: str, level: int = logging.WARNING) -> None: """ self._logger.log(level, message) - async def open(self) -> bool: # noqa: A003 + async def open(self) -> bool: """ Open the Endpoint. diff --git a/ipv8/messaging/payload_dataclass.py b/ipv8/messaging/payload_dataclass.py index 5d13629ad..5b8290b8b 100644 --- a/ipv8/messaging/payload_dataclass.py +++ b/ipv8/messaging/payload_dataclass.py @@ -1,8 +1,9 @@ from __future__ import annotations +from collections.abc import Iterable from dataclasses import dataclass as ogdataclass from functools import partial -from typing import Callable, Iterable, Type, TypeVar, cast, get_type_hints +from typing import Callable, TypeVar, cast, get_args, get_type_hints from .lazy_payload import VariablePayload, vp_compile from .serialization import FormatListType, Serializable @@ -17,7 +18,7 @@ def type_from_format(fmt: str) -> TypeVar: return out -def type_map(t: Type) -> FormatListType: # noqa: PLR0911 +def type_map(t: type) -> FormatListType: # noqa: PLR0911 if t is bool: return "?" if t is int: @@ -31,12 +32,12 @@ def type_map(t: Type) -> FormatListType: # noqa: PLR0911 if isinstance(t, TypeVar): return t.__name__ if getattr(t, '__origin__', None) in (tuple, list, set): - fmt = t.__args__[0] + fmt = get_args(t)[0] if issubclass(fmt, Serializable): return [fmt] - return f"arrayH-{type_map(t.__args__[0])}" + return f"arrayH-{type_map(fmt)}" if isinstance(t, (tuple, list, set)) or Serializable in getattr(t, "mro", list)(): - return cast(Type[Serializable], t) + return cast(type[Serializable], t) raise NotImplementedError(t, " unknown") @@ -47,7 +48,7 @@ def dataclass(cls: type | None = None, *, # noqa: PLR0913 order: bool = False, unsafe_hash: bool = False, frozen: bool = False, - msg_id: int | None = None) -> partial[Type[VariablePayload]] | Type[VariablePayload]: + msg_id: int | None = None) -> partial[type[VariablePayload]] | type[VariablePayload]: """ Equivalent to ``@dataclass``, but also makes the wrapped class a ``VariablePayload``. @@ -55,7 +56,7 @@ def dataclass(cls: type | None = None, *, # noqa: PLR0913 """ if cls is None: # Forward user parameters. Format: ``@dataclass(foo=bar)``. - return partial(cast(Callable[..., Type[VariablePayload]], dataclass), init=init, repr=repr, eq=eq, order=order, + return partial(cast(Callable[..., type[VariablePayload]], dataclass), init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, msg_id=msg_id) # Finally, we have the actual class. Format: ``@dataclass`` or forwarded from partial (see above). diff --git a/ipv8/messaging/serialization.py b/ipv8/messaging/serialization.py index 7d0e15756..efd600f3e 100644 --- a/ipv8/messaging/serialization.py +++ b/ipv8/messaging/serialization.py @@ -19,7 +19,7 @@ ADDRESS_TYPE_IPV6 = 0x03 -FormatListType = typing.Union[str, typing.Type["Serializable"], typing.List["FormatListType"]] +FormatListType = typing.Union[str, type["Serializable"], list["FormatListType"]] class PackError(RuntimeError): @@ -539,7 +539,7 @@ def unpack_serializable(self, try: offset = self._packers[fmt].unpack(data, offset, unpack_list) # type: ignore[index] except KeyError: - fmt = cast(typing.Type, fmt) # If this is not a type, we'll crash + fmt = cast(type, fmt) # If this is not a type, we'll crash if not issubclass(fmt, Serializable): raise offset = self._packers['payload'].unpack(data, offset, unpack_list, fmt) diff --git a/ipv8/peerdiscovery/community.py b/ipv8/peerdiscovery/community.py index a8840f94c..a8ed07c68 100644 --- a/ipv8/peerdiscovery/community.py +++ b/ipv8/peerdiscovery/community.py @@ -3,7 +3,7 @@ from binascii import unhexlify from random import sample from time import time -from typing import TYPE_CHECKING, Sequence, Union, cast +from typing import TYPE_CHECKING, Union, cast from ..community import Community, CommunitySettings from ..keyvault.crypto import default_eccrypto @@ -24,6 +24,8 @@ ) if TYPE_CHECKING: + from collections.abc import Sequence + from ..types import Address diff --git a/ipv8/peerdiscovery/discovery.py b/ipv8/peerdiscovery/discovery.py index 586ba7d79..ead5e4b35 100644 --- a/ipv8/peerdiscovery/discovery.py +++ b/ipv8/peerdiscovery/discovery.py @@ -129,7 +129,7 @@ def get_available_root(self) -> Peer | None: available = list(set(self._neighborhood) - set(self.under_construction.keys())) return choice(available) if available else None - def take_step(self) -> None: # noqa: C901 + def take_step(self) -> None: # noqa: C901, PLR0912 """ Attempt to grow an edge. """ diff --git a/ipv8/peerdiscovery/network.py b/ipv8/peerdiscovery/network.py index 9b73f0aa0..5c74cd482 100644 --- a/ipv8/peerdiscovery/network.py +++ b/ipv8/peerdiscovery/network.py @@ -6,19 +6,21 @@ from collections import OrderedDict from operator import methodcaller from threading import RLock -from typing import TYPE_CHECKING, Iterable, NamedTuple, Set, cast +from typing import TYPE_CHECKING, NamedTuple, cast from ..messaging.serialization import default_serializer from ..types import Address if TYPE_CHECKING: + from collections.abc import Iterable + from ..types import Overlay, Peer MID = bytes PublicKeyMat = bytes Service = bytes -ServiceSet = Set[Service] +ServiceSet = set[Service] class WalkableAddress(NamedTuple): diff --git a/ipv8/requestcache.py b/ipv8/requestcache.py index a1f1e108f..28069bdc1 100644 --- a/ipv8/requestcache.py +++ b/ipv8/requestcache.py @@ -7,12 +7,15 @@ from contextlib import contextmanager, suppress from random import random from threading import Lock -from typing import Generator, Iterable, TypeVar, overload +from typing import TYPE_CHECKING, TypeVar, overload from typing_extensions import Protocol from .taskmanager import TaskManager +if TYPE_CHECKING: + from collections.abc import Generator, Iterable + class NumberCache: """ @@ -27,7 +30,8 @@ def __init__(self, request_cache: RequestCache, prefix: str, number: int) -> Non self._logger = logging.getLogger(self.__class__.__name__) if request_cache.has(prefix, number): - raise RuntimeError("This number is already in use '%s'" % number) + msg = f"This number is already in use '{number}'" + raise RuntimeError(msg) self._prefix = prefix self._number = number diff --git a/ipv8/taskmanager.py b/ipv8/taskmanager.py index 4e39eee6c..ec7bd2d19 100644 --- a/ipv8/taskmanager.py +++ b/ipv8/taskmanager.py @@ -3,16 +3,18 @@ import logging import time import traceback +import types from asyncio import CancelledError, Future, Task, ensure_future, gather, get_running_loop, iscoroutinefunction, sleep from contextlib import suppress from functools import wraps from threading import RLock -from typing import TYPE_CHECKING, Any, Callable, Coroutine, Hashable, Sequence +from typing import TYPE_CHECKING, Any, Callable from weakref import WeakValueDictionary from .util import coroutine, succeed if TYPE_CHECKING: + from collections.abc import Coroutine, Hashable, Sequence from concurrent.futures import ThreadPoolExecutor MAX_TASK_AGE = 600 @@ -55,6 +57,31 @@ def wrapper(self: TaskManager, *args: Any, **kwargs: Any) -> Future: # noqa: AN return wrapper +def set_name(self: Future, __value: object) -> None: + """ + This method mimics ``Task.set_name`` for non-Task objects. See ``Task.set_name`` for signature description. + + :param self: The ``Future`` instance to set the name of + :param __value: The value to set + + .. seealso:: :class:`asyncio.Task` + .. seealso:: :func:`asyncio.Task.set_name` + """ + self.name = __value # type: ignore[attr-defined] + + +def get_name(self: Future) -> object: + """ + This method mimics ``Task.get_name`` for non-Task objects. See ``Task.get_name`` for signature description. + + :param self: The ``Future`` instance to get the name of + + .. seealso:: :class:`asyncio.Task` + .. seealso:: :func:`asyncio.Task.get_name` + """ + return self.name # type: ignore[attr-defined] + + class TaskManager: """ Provides a set of tools to maintain a list of asyncio Tasks that are to be @@ -122,7 +149,8 @@ def register_task(self, name: Hashable, task: Callable | Coroutine | Future, # return succeed(None) if self.is_pending_task_active(name): - raise RuntimeError("Task already exists: '%s'" % name) + msg = f"Task already exists: '{name}'" + raise RuntimeError(msg) if callable(task): task = task if iscoroutinefunction(task) else coroutine(task) @@ -138,12 +166,10 @@ def register_task(self, name: Hashable, task: Callable | Coroutine | Future, # # in _pending_tasks. Instead, we add them as attributes to the task. task.start_time = time.time() # type: ignore[attr-defined] task.interval = interval # type: ignore[attr-defined] - # The set_name function is only available in Python 3.8+ - task_name = f"{self.__class__.__name__}:{name}" - if hasattr(task, "set_name"): - task.set_name(task_name) - else: - task.name = task_name # type: ignore[attr-defined] + if not hasattr(task, "set_name"): + task.set_name = types.MethodType(set_name, task) # type: ignore[attr-defined] + task.get_name = types.MethodType(get_name, task) # type: ignore[attr-defined] + task.set_name(f"{self.__class__.__name__}:{name}") # type: ignore[attr-defined] assert isinstance(task, (Task, Future)) diff --git a/ipv8/test/REST/rest_base.py b/ipv8/test/REST/rest_base.py index ae75eb793..e7a8582b7 100644 --- a/ipv8/test/REST/rest_base.py +++ b/ipv8/test/REST/rest_base.py @@ -2,7 +2,7 @@ from asyncio import Future, Transport from threading import RLock -from typing import TYPE_CHECKING, Any, Awaitable, Callable, cast +from typing import TYPE_CHECKING, Any, Callable, cast from aiohttp import BaseConnector, ClientRequest, ClientSession, ClientTimeout, web from aiohttp.client_proto import ResponseHandler @@ -23,6 +23,7 @@ from ..base import TestBase if TYPE_CHECKING: + from collections.abc import Awaitable from ssl import SSLContext from aiohttp.tracing import Trace diff --git a/ipv8/test/REST/test_attestation_endpoint.py b/ipv8/test/REST/test_attestation_endpoint.py deleted file mode 100644 index bf0e94eec..000000000 --- a/ipv8/test/REST/test_attestation_endpoint.py +++ /dev/null @@ -1,412 +0,0 @@ -from __future__ import annotations - -from asyncio import sleep -from base64 import b64encode -from typing import Collection, Sequence - -from ...attestation.identity.community import IdentityCommunity, IdentitySettings -from ...attestation.identity.manager import IdentityManager -from ...attestation.wallet.community import AttestationCommunity, AttestationSettings -from ..REST.rest_base import MockRestIPv8, RESTTestBase - - -class TestAttestationEndpoint(RESTTestBase): - """ - Class for testing the REST API of the AttestationEndpoint. - """ - - async def setUp(self) -> None: - """ - Create a new memory-based identity manager and set up the necessary communities. - """ - super().setUp() - identity_manager = IdentityManager(":memory:") - await self.initialize([AttestationCommunity, IdentityCommunity], 2, - settings=[AttestationSettings(working_directory=':memory:'), - IdentitySettings(identity_manager=identity_manager)]) - - async def make_outstanding(self, node: MockRestIPv8) -> list[Sequence[str, str, str]]: - """ - Forward a request for outstanding attestation requests. - """ - return await self.make_request(node, 'attestation', 'get', {'type': 'outstanding'}) - - async def make_verification_output(self, node: MockRestIPv8) -> dict[str, list[Sequence[str, float]]]: - """ - Forward a request for the verification outputs. - """ - return await self.make_request(node, 'attestation', 'get', {'type': 'verification_output'}) - - async def make_peers(self, node: MockRestIPv8) -> list[str]: - """ - Forward a request for the known peers in the network. - """ - return await self.make_request(node, 'attestation', 'get', {'type': 'peers'}) - - async def make_attributes(self, node: MockRestIPv8) -> list[Sequence[str, str, dict[str, str | int], str]]: - """ - Forward a request for the attributes of a peer. - """ - return await self.make_request(node, 'attestation', 'get', {'type': 'attributes'}) - - async def wait_for_attributes(self, node: MockRestIPv8) -> list[Sequence[str, str, dict[str, str | int], str]]: - """ - Forward a request for the attributes of a peer. - """ - attributes = await self.make_request(node, 'attestation', 'get', {'type': 'attributes'}) - while not attributes: - attributes = await self.make_request(node, 'attestation', 'get', {'type': 'attributes'}) - await self.deliver_messages() - return attributes - - async def make_drop_identity(self, node: MockRestIPv8) -> dict[str, bool]: - """ - Forward a request for dropping a peer's identity. - """ - return await self.make_request(node, 'attestation', 'get', {'type': 'drop_identity'}) - - async def make_outstanding_verify(self, node: MockRestIPv8) -> list[Sequence[str, str]]: - """ - Forward a request which requests information on the outstanding verify requests. - """ - return await self.make_request(node, 'attestation', 'get', {'type': 'outstanding_verify'}) - - async def make_attestation_request(self, node: MockRestIPv8, attribute_name: str, mid: str, - metadata: dict | None = None) -> dict[str, bool]: - """ - Forward a request for the attestation of an attribute. - """ - # Add the type of the request (request), and the rest of the parameters - request_parameters = {'type': 'request', - 'id_format': 'id_metadata', - 'attribute_name': attribute_name, - 'mid': mid} - if metadata: - request_parameters['metadata'] = metadata - return await self.make_request(node, 'attestation', 'post', request_parameters) - - async def make_attest(self, node: MockRestIPv8, attribute_name: str, attribute_value: list[str], - mid: str) -> dict[str, bool]: - """ - Forward a request which attests an attestation request. - """ - return await self.make_request(node, 'attestation', 'post', {'type': 'attest', - 'attribute_name': attribute_name, - 'attribute_value': attribute_value, - 'mid': mid}) - - async def make_verify(self, node: MockRestIPv8, attribute_hash: str, attribute_values: str, - mid: str) -> dict[str, bool]: - """ - Forward a request which demands the verification of an attestation. - """ - return await self.make_request(node, 'attestation', 'post', {'type': 'verify', - 'attribute_hash': attribute_hash, - 'mid': mid, - 'attribute_values': attribute_values}) - - async def make_allow_verify(self, node: MockRestIPv8, attribute_name: str, mid: str) -> dict[str, bool]: - """ - Forward a request which requests that verifications be allowed for a particular peer for a particular attribute. - """ - return await self.make_request(node, 'attestation', 'post', {'type': 'allow_verify', - 'attribute_name': attribute_name, - 'mid': mid}) - - async def create_attestation_request(self, node: MockRestIPv8, attribute_name: str, - metadata: dict | None = None) -> None: - """ - Request all known peers to attest to the node's given attribute name. - """ - peer_list = await self.wait_for_peers(node) - for mid in peer_list: - await self.make_attestation_request(node, attribute_name, mid, metadata=metadata) - - async def wait_for_peers(self, node: MockRestIPv8) -> list[str]: - """ - Wait until this peer receives a non-empty list of fellow peers in the network. - """ - peer_list = await self.make_peers(node) - while not peer_list: - await sleep(.1) - peer_list = await self.make_peers(node) - return peer_list - - async def wait_for_outstanding_requests(self, node: MockRestIPv8) -> list[tuple[bytes, str, str]]: - """ - Wait until this peer receives a non-empty list of outstanding attestation requests. - """ - outstanding_requests = await self.make_outstanding(node) - while not outstanding_requests: - await sleep(.1) - outstanding_requests = await self.make_outstanding(node) - return [(x[0].encode('utf-8'), x[1], x[2]) for x in outstanding_requests] - - async def attest_all_outstanding_requests(self, node: MockRestIPv8, attribute_name: str, - attribute_value: str) -> tuple[list[tuple[bytes, str, str]], - list[dict[str, bool]]]: - """ - Forward an attestation for each of the outstanding attestation requests. - - :return: a list of the outstanding requests and their (empty if successful) request responses - """ - outstanding_requests = await self.wait_for_outstanding_requests(node) - self.assertFalse(outstanding_requests == [], "Something went wrong, no request was received.") - - # Collect the responses of the attestations; if functioning properly, this should be a list of empty strings - responses = [] - - for outstanding_request in outstanding_requests: - # The attestation value is already computed, so don't bother recomputing it here - mid = outstanding_request[0].decode('utf-8') - attribute_value = b64encode(attribute_value).decode('utf-8')\ - if isinstance(attribute_value, bytes) else attribute_value - response = await self.make_attest(node, attribute_name, attribute_value, mid) - responses.append(response) - - return outstanding_requests, responses - - async def verify_all_attestations(self, node: MockRestIPv8, peer_mids: Collection[bytes], attribute_hash: str, - attribute_values: str) -> list[dict[str, bool]]: - """ - Forward an attestation verification for a set of attestations. - - :param peer_mids: the set of peer mids to which a verification request will be sent - :return: the verification responses, as returned by the well-known peer. Ideally these should be all empty - """ - assert peer_mids, "Attestation list is empty" - - verification_responses = [] - - for mid in peer_mids: - decoded_mid = b64encode(mid).decode('utf-8') if isinstance(mid, bytes) else mid - intermediary_response = await self.make_verify(node, attribute_hash, attribute_values, decoded_mid) - verification_responses.append(intermediary_response) - - return verification_responses - - async def test_get_peers_request(self) -> None: - """ - Test the (GET: peers request) type. - """ - await self.introduce_nodes() - other_peer_mids = [b64encode(self.mid(1)).decode('utf-8')] - result = await self.wait_for_peers(self.node(0)) - self.assertTrue(any(x in other_peer_mids for x in result), "Could not find the second peer.") - - async def test_get_outstanding_requests(self) -> None: - """ - Test the (GET: outstanding) request type. - """ - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - - result = await self.wait_for_outstanding_requests(self.node(0)) - - mid = b64encode(self.mid(1)) - self.assertTrue(any(x[0] == mid and x[1] == 'QR' for x in result), - "Could not find the outstanding request forwarded by the second peer") - - async def test_get_verification_output(self) -> None: - """ - Test the (GET: verification output) request type. - """ - # Forward the attestations to the well-known peer - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - await self.attest_all_outstanding_requests(self.node(0), 'QR', 'data') - - # Get the hash of the attestation to be validated (the one which was just attested) - attributes = await self.wait_for_attributes(self.node(1)) - attribute_hash = attributes[0][1] - - # Forward the actual verification - verification_responses = await self.verify_all_attestations(self.node(0), - [self.mid(1)], - attribute_hash, 'YXNk,YXNkMg==') - self.assertTrue(all("success" in x and x["success"] for x in verification_responses), - "At least one of the verification responses was non-empty.") - - # Unlock the verification - outstanding_verifications = [] - while not outstanding_verifications: - outstanding_verifications = await self.make_outstanding_verify(self.node(1)) - self.assertIsNotNone(outstanding_verifications, "Could not retrieve any outstanding verifications") - await self.deliver_messages() - - mid = outstanding_verifications[0][0] - await self.make_allow_verify(self.node(1), 'QR', mid) - await sleep(.1) - - # Get the output - verification_output = await self.make_verification_output(self.node(0)) - self.assertTrue([["YXNk", 0.0], ["YXNkMg==", 0.0]] in verification_output.values(), - "Something went wrong with the verification. Unexpected output values.") - - async def test_get_outstanding_verify(self) -> None: - """ - Test the (GET: outstanding verify) request type. - """ - # Forward the attestations to the well-known peer - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - await self.attest_all_outstanding_requests(self.node(0), 'QR', 'data') - - # Get the hash of the attestation to be validated (the one which was just attested) - attributes = await self.wait_for_attributes(self.node(1)) - attribute_hash = attributes[0][1] - - # Forward the actual verification - verification_responses = await self.verify_all_attestations(self.node(0), - [self.mid(1)], - attribute_hash, 'YXNk,YXNkMg==') - self.assertTrue(all("success" in x and x["success"] for x in verification_responses), - "At least one of the verification responses was non-empty.") - - # Unlock the verification - outstanding_verifications = [] - while not outstanding_verifications: - outstanding_verifications = await self.make_outstanding_verify(self.node(1)) - self.assertIsNotNone(outstanding_verifications, "Could not retrieve any outstanding verifications") - await self.deliver_messages() - - # Retrieve only the mids - result = [x[0] for x in outstanding_verifications] - self.assertTrue(any(x in result for x in [b64encode(self.mid(0)).decode('utf-8')]), - "Something went wrong. Could not find a master peer mid in the " - "outstanding verification requests.") - - async def test_get_attributes(self) -> None: - """ - Test the (GET: attributes) request type. - """ - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - await self.attest_all_outstanding_requests(self.node(0), 'QR', 'data') - - # Get the hash of the attestation to be validated (the one which was just attested) - attributes = await self.wait_for_attributes(self.node(1)) - self.assertTrue(attributes[0][0] == 'QR' and attributes[0][1] != "", - "The response was not as expected. This would suggest that something went wrong with " - "the attributes request.") - - async def test_get_drop_identity(self) -> None: - """ - Test the (GET: drop identity) request type. - """ - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - outstanding_requests = await self.wait_for_outstanding_requests(self.node(0)) - self.assertFalse(outstanding_requests == [], "The attestation requests were not received.") - - # Ensure that no block/attestation exists - attributes = await self.make_attributes(self.node(1)) - self.assertEqual(attributes, [], "Something's wrong, there shouldn't be any blocks.") - - # Attest the outstanding request. This should mean that the attribute DB is non-empty in the well-known peer - await self.attest_all_outstanding_requests(self.node(0), 'QR', 'data') - - # Ensure that the attestation has been completed - attributes = await self.wait_for_attributes(self.node(1)) - self.assertNotEqual(attributes, [], "Something's wrong, the attribute list should be non-empty.") - - # Drop the identity - result = await self.make_drop_identity(self.node(0)) - self.assertIn("success", result, "The identity could not be dropped. Success parameter not in response.") - self.assertTrue(result["success"], "The identity could not be dropped, not successful.") - - # Make sure the identity was successfully dropped - result = await self.make_attributes(self.node(0)) - self.assertEqual(result, [], 'The identity could not be dropped. Block DB still populated.') - - result = await self.make_outstanding(self.node(0)) - self.assertEqual(result, [], 'The identity could not be dropped. Outstanding requests still remaining.') - - async def test_post_attestation_request(self) -> None: - """ - Test the (POST: request) request type. - """ - # This should return an empty response - outstanding_requests = await self.make_outstanding(self.node(0)) - self.assertEqual(outstanding_requests, [], "Something went wrong, there should be no outstanding requests.") - - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - - # This should return a non-empty response - outstanding_requests = await self.wait_for_outstanding_requests(self.node(0)) - self.assertFalse(outstanding_requests == [], "Something went wrong, no request was received.") - - async def test_post_attest(self) -> None: - """ - Test the (POST: attest) request type. - """ - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - - attributes = await self.make_attributes(self.node(1)) - self.assertTrue(len(attributes) == 0, "There mustn't already be any attestations in the other peer.") - - responses = await self.attest_all_outstanding_requests(self.node(0), 'QR', 'data') - request_responses = list(responses[1]) - self.assertTrue(all("success" in x and x["success"] for x in request_responses), - "Something went wrong, not all responses were successful.") - - attributes = await self.wait_for_attributes(self.node(1)) - self.assertTrue(len(attributes) == 1, "There should only be one attestation in the DB.") - self.assertTrue(attributes[0][0] == 'QR', f"Expected attestation for QR, got it for {attributes[0][0]}") - - attributes = await self.make_attributes(self.node(0)) - self.assertTrue(len(attributes) == 0, "There should be no attribute in the DB of the attester.") - - async def test_post_verify(self) -> None: - """ - Test the (POST: verify) request type. - """ - # Forward the attestations to the well-known peer - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - await self.attest_all_outstanding_requests(self.node(0), 'QR', 'data') - - # Get the hash of the attestation to be validated (the one which was just attested) - attributes = await self.wait_for_attributes(self.node(1)) - attribute_hash = attributes[0][1] - - # Forward the actual verification - verification_responses = await self.verify_all_attestations(self.node(0), - [self.mid(1)], - attribute_hash, 'YXNk,YXNkMg==') - self.assertTrue(all("success" in x and x["success"] for x in verification_responses), - "At least one of the verification responses was non-empty.") - - async def test_post_allow_verify(self) -> None: - """ - Test the (POST: allow verify) request type. - """ - # Forward the attestations to the well-known peer - await self.introduce_nodes() - await self.create_attestation_request(self.node(1), 'QR') - await self.attest_all_outstanding_requests(self.node(0), 'QR', 'data') - - # Get the hash of the attestation to be validated (the one which was just attested) - attributes = await self.wait_for_attributes(self.node(1)) - attribute_hash = attributes[0][1] - - # Forward the actual verification - verification_responses = await self.verify_all_attestations(self.node(0), - [self.mid(1)], - attribute_hash, 'YXNk,YXNkMg==') - self.assertTrue(all("success" in x and x["success"] for x in verification_responses), - "At least one of the verification responses was non-empty.") - - # Unlock the verification - outstanding_verifications = [] - while not outstanding_verifications: - outstanding_verifications = await self.make_outstanding_verify(self.node(1)) - self.assertIsNotNone(outstanding_verifications, "Could not retrieve any outstanding verifications") - await self.deliver_messages() - - mid = outstanding_verifications[0][0] - response = await self.make_allow_verify(self.node(1), 'QR', mid) - self.assertIn("success", response, "The attestion could not be unlocked: success not in JSON response") - self.assertTrue(response["success"], "The attestation could not be unlocked: not successful.") diff --git a/ipv8/test/attestation/wallet/test_attestation_community.py b/ipv8/test/attestation/wallet/test_attestation_community.py index ad25384a2..8a43d3fe2 100644 --- a/ipv8/test/attestation/wallet/test_attestation_community.py +++ b/ipv8/test/attestation/wallet/test_attestation_community.py @@ -201,7 +201,7 @@ async def test_verify_attestation(self) -> None: """ serialized = "" filename = os.path.join(os.path.dirname(__file__), 'attestation.txt') - with open(filename) as f: # noqa: ASYNC101 + with open(filename) as f: # noqa: ASYNC230 serialized = unhexlify(f.read().strip()) attestation = BonehAttestation.unserialize(serialized, "id_metadata") attestation_hash = unhexlify('9019195eb75c07ec3e86a62c314dcf5ef2bbcc0d') @@ -270,7 +270,7 @@ async def test_verify_attestation_big(self) -> None: Check if an attestation can be verified for id_metadata_big. """ filename = os.path.join(os.path.dirname(__file__), 'attestation_big.txt') - with open(filename) as f: # noqa: ASYNC101 + with open(filename) as f: # noqa: ASYNC230 serialized = unhexlify(f.read().strip()) attestation = BonehAttestation.unserialize(serialized, "id_metadata_big") attestation_hash = unhexlify('113d31c31b626268a16c198cbd58dd5aa8d1d81c') @@ -298,7 +298,7 @@ async def test_verify_attestation_range(self) -> None: Check if an attestation can be verified for id_metadata_range_18plus. """ filename = os.path.join(os.path.dirname(__file__), 'attestation_range.txt') - with open(filename) as f: # noqa: ASYNC101 + with open(filename) as f: # noqa: ASYNC230 serialized = unhexlify(f.read().strip()) attestation = PengBaoAttestation.unserialize_private(self.private_key, serialized, "id_metadata_range_18plus") attestation_hash = unhexlify('b40c8734ba6c91a49670c1f0152c7f4dac2a8272') diff --git a/ipv8/test/base.py b/ipv8/test/base.py index 79eff3046..35e2f6113 100644 --- a/ipv8/test/base.py +++ b/ipv8/test/base.py @@ -9,9 +9,10 @@ import unittest import uuid from asyncio import AbstractEventLoop, Task, all_tasks, ensure_future, get_running_loop, iscoroutine, sleep +from collections.abc import Awaitable, Coroutine from contextlib import contextmanager from functools import partial -from typing import TYPE_CHECKING, Awaitable, Callable, Coroutine, Generic, Type, TypeVar, cast +from typing import TYPE_CHECKING, Callable, Generic, TypeVar, cast from ..lazy_community import PacketDecodingError, lazy_wrapper, lazy_wrapper_unsigned from ..messaging.interfaces.lan_addresses.interfaces import get_providers @@ -115,26 +116,16 @@ def trap(_: Community, sender: Peer, *payloads: Payload) -> None: self.received_messages += list(trap.payloads) -if sys.version_info[0] == 3 and sys.version_info[1] <= 7: # noqa: YTT201, YTT203 - # unittest.IsolatedAsyncioTestCase does not exist below Python 3.8, use asynctest instead - import asynctest - TestCaseClass = asynctest.TestCase - USE_ASYNC_TEST = True -else: - TestCaseClass = unittest.IsolatedAsyncioTestCase - USE_ASYNC_TEST = False - OT = TypeVar("OT", bound=Overlay) -class TestBase(TestCaseClass, Generic[OT]): +class TestBase(unittest.IsolatedAsyncioTestCase, Generic[OT]): """ Base TestCase to allow easy testing of IPv8 overlays. """ __testing__ = True __lockup_timestamp__ = 0 - __asynctest_compatibility_mode__ = USE_ASYNC_TEST # The time after which the whole test suite is os.exited MAX_TEST_TIME = 10 @@ -152,29 +143,25 @@ def __init__(self, methodName: str = 'runTest') -> None: # noqa: N803 self._uncaught_async_failure = None @property - def __loop(self) -> AbstractEventLoop: + def loop(self) -> AbstractEventLoop: """ Return the asyncio event loop used for the test case. """ if hasattr(self, "_asyncioTestLoop"): - # Python 3.8, 3.9, and 3.10. + # Python 3.9, and 3.10. return self._asyncioTestLoop # Python 3.11 (and up?). return self._asyncioRunner.get_loop() - loop = getattr(TestCaseClass, "loop", __loop) - def __call_internal_in_context(self, func: Callable | Coroutine) -> None: """ Call setUp or tearDown within the unittest.IsolatedAsyncioTestCase context. - - This is not used for asynctest compatibility mode. """ if hasattr(self, "_asyncioTestContext"): # Python 3.11 (and up?). self._asyncioTestContext.run(func) else: - # Python 3.8, 3.9, and 3.10. + # Python 3.9, and 3.10. self._callTestMethod(func) def _callSetUp(self) -> None: # noqa: N802 @@ -273,17 +260,12 @@ async def tearDown(self) -> None: while self._tempdirs: shutil.rmtree(self._tempdirs.pop(), ignore_errors=True) # Now that everyone has calmed down, sweep up the remaining callbacks and check if they failed. - # [port] ``asynctest.helpers.exhaust_callbacks`` no longer works in Python 3.10 - while self.loop._ready: # noqa: SLF001 + while self.loop._ready: # noqa: SLF001, ASYNC110 await sleep(0) - # [end of ``asynctest.helpers.exhaust_callbacks`` port] if self._uncaught_async_failure is not None: raise self._uncaught_async_failure["exception"] self.loop.set_exception_handler(None) # None is equivalent to the default handler - if self.__asynctest_compatibility_mode__: - super().tearDown() - else: - await super().asyncTearDown() + await super().asyncTearDown() @classmethod def setUpClass(cls: TestBase) -> None: # noqa: C901 @@ -318,7 +300,7 @@ def check_loop() -> None: if tasks: print("Pending tasks:") # noqa: T201 for task in tasks: - print("> %s" % task) # noqa: T201 + print(f"> {task}") # noqa: T201 except RuntimeError: print("Failed to acquire the pending tasks. Your event loop may already be closed!") # noqa: T201 @@ -348,7 +330,7 @@ def create_node(self, settings: CommunitySettings | None = None, create_dht: boo """ Create a new IPv8-like container to store node related information. """ - return MockIPv8("low", cast(Type[Community], self.overlay_class), settings, create_dht, enable_statistics) + return MockIPv8("low", cast(type[Community], self.overlay_class), settings, create_dht, enable_statistics) def add_node_to_experiment(self, node: MockIPv8) -> None: """ @@ -366,9 +348,7 @@ def is_background_task(task: Task) -> bool: """ Check if the given task is to be ignored. """ - # Only in Python 3.8+ will we have a get_name function - name = task.get_name() if hasattr(task, 'get_name') else getattr(task, 'name', f'Task-{id(task)}') - return name.endswith('_check_tasks') + return task.get_name().endswith('_check_tasks') async def deliver_messages(self, timeout: float = .1) -> None: """ diff --git a/ipv8/test/dht/test_community.py b/ipv8/test/dht/test_community.py index 7a4adcdd7..c559e9dc3 100644 --- a/ipv8/test/dht/test_community.py +++ b/ipv8/test/dht/test_community.py @@ -1,6 +1,6 @@ import time from asyncio import TimeoutError, ensure_future, wait_for -from typing import Iterable +from collections.abc import Iterable from ...dht import DHTError from ...dht.community import DHTCommunity diff --git a/ipv8/test/dht/test_routing.py b/ipv8/test/dht/test_routing.py index f98cec015..8fb415a56 100644 --- a/ipv8/test/dht/test_routing.py +++ b/ipv8/test/dht/test_routing.py @@ -17,13 +17,13 @@ def __init__(self, binary_prefix: str) -> None: """ id_binary = int(binary_prefix + '0' * (160 - len(binary_prefix)), 2) super().__init__(LibNaCLSK(b'LibNaCLSK:' + id_binary.to_bytes(64, "big")), ('1.1.1.1', 1)) - id_hex = '%x' % id_binary + id_hex = f'{id_binary:x}' id_hex = '0' + id_hex if len(id_hex) % 2 != 0 else id_hex self._id = unhexlify(id_hex) @property - def id(self) -> bytes: # noqa: A003 + def id(self) -> bytes: """ Our fixed testing id. """ diff --git a/ipv8/test/messaging/anonymization/test_community.py b/ipv8/test/messaging/anonymization/test_community.py index bb307d618..b594dcf21 100644 --- a/ipv8/test/messaging/anonymization/test_community.py +++ b/ipv8/test/messaging/anonymization/test_community.py @@ -1,8 +1,9 @@ from __future__ import annotations from asyncio import Future, ensure_future, iscoroutine +from collections.abc import Awaitable from functools import partial -from typing import TYPE_CHECKING, Awaitable, cast +from typing import TYPE_CHECKING, cast from unittest.mock import Mock from ....messaging.anonymization.community import TunnelCommunity, TunnelSettings diff --git a/ipv8/test/messaging/interfaces/lan_addresses/test_addressprovider.py b/ipv8/test/messaging/interfaces/lan_addresses/test_addressprovider.py index 863d51b9b..9c270fb1a 100644 --- a/ipv8/test/messaging/interfaces/lan_addresses/test_addressprovider.py +++ b/ipv8/test/messaging/interfaces/lan_addresses/test_addressprovider.py @@ -4,7 +4,6 @@ import io from asyncio import sleep from time import time -from typing import Set from .....messaging.interfaces.lan_addresses.addressprovider import AddressProvider from ....base import TestBase @@ -15,7 +14,7 @@ class ErroringProvider(AddressProvider): A provider that errors out when getting addresses. """ - def get_addresses(self) -> Set[str]: + def get_addresses(self) -> set[str]: """ Raise and set an exception. """ @@ -39,7 +38,7 @@ def __init__(self) -> None: super().__init__() self.invocations = 0 - def get_addresses(self) -> Set[str]: + def get_addresses(self) -> set[str]: """ Add to the count and return no addresses. """ @@ -97,7 +96,7 @@ async def test_get_addresses_buffered_timeout(self) -> None: provider = InvocationCountingProvider() provider.get_addresses_buffered() - while time() - provider.addresses_ts <= 0.01: + while time() - provider.addresses_ts <= 0.01: # noqa: ASYNC110 await sleep(0.01) # 0.01 seconds should have passed, so the addresses should be re-discovered diff --git a/ipv8/test/messaging/interfaces/lan_addresses/test_interfaces.py b/ipv8/test/messaging/interfaces/lan_addresses/test_interfaces.py index 4270bfd3b..a75a7d341 100644 --- a/ipv8/test/messaging/interfaces/lan_addresses/test_interfaces.py +++ b/ipv8/test/messaging/interfaces/lan_addresses/test_interfaces.py @@ -1,25 +1,28 @@ from __future__ import annotations -from typing import Collection, Set +from typing import TYPE_CHECKING from .....messaging.interfaces.lan_addresses.addressprovider import AddressProvider from .....messaging.interfaces.lan_addresses.interfaces import get_lan_addresses from ....base import TestBase +if TYPE_CHECKING: + from collections.abc import Collection + class PresetProvider(AddressProvider): """ Provider that returns a specific value. """ - def __init__(self, return_value: Set[str]) -> None: + def __init__(self, return_value: set[str]) -> None: """ Create a new provider with a preset return value. """ super().__init__() self.return_value = return_value - def get_addresses(self) -> Set[str]: + def get_addresses(self) -> set[str]: """ Return our preset return value. """ @@ -38,7 +41,7 @@ def __init__(self) -> None: super().__init__() self.return_values = [] - def set_return_values(self, return_values: Collection[Set[str]]) -> None: + def set_return_values(self, return_values: Collection[set[str]]) -> None: """ Initialize providers for the given return values. """ diff --git a/ipv8/test/messaging/interfaces/udp/test_endpoint.py b/ipv8/test/messaging/interfaces/udp/test_endpoint.py index 4cde462a5..b8a407cd0 100644 --- a/ipv8/test/messaging/interfaces/udp/test_endpoint.py +++ b/ipv8/test/messaging/interfaces/udp/test_endpoint.py @@ -80,7 +80,7 @@ async def test_send_many_messages(self) -> None: # range must be in [1, 51), since Asyncio transports discard empty datagrams for ind in range(1, 51): self.endpoint1.send(self.ep2_address, b'a' * ind) - while len(self.endpoint2_listener.incoming) < 50: + while len(self.endpoint2_listener.incoming) < 50: # noqa: ASYNC110 await sleep(.02) self.assertEqual(len(self.endpoint2_listener.incoming), 50) diff --git a/ipv8/test/messaging/test_payload_dataclass.py b/ipv8/test/messaging/test_payload_dataclass.py index dcd711a00..2ca9e5938 100644 --- a/ipv8/test/messaging/test_payload_dataclass.py +++ b/ipv8/test/messaging/test_payload_dataclass.py @@ -2,7 +2,7 @@ from dataclasses import dataclass as ogdataclass from dataclasses import is_dataclass -from typing import List, TypeVar +from typing import TypeVar from ...messaging.payload_dataclass import dataclass, type_from_format from ...messaging.serialization import default_serializer @@ -73,7 +73,7 @@ class NestedListType: A single list of nested payload. """ - a: List[NativeInt] # Backward compatibility: Python >= 3.9 can use ``list[NativeInt]`` + a: list[NativeInt] @dataclass class ListIntType: @@ -81,7 +81,7 @@ class ListIntType: A single list of integers. """ - a: List[int] + a: list[int] @dataclass class ListBoolType: @@ -89,7 +89,7 @@ class ListBoolType: A single list of booleans. """ - a: List[bool] + a: list[bool] @ogdataclass class Unknown: @@ -174,10 +174,10 @@ class Everything: b: bytes c: varlenH d: EverythingItem - e: List[EverythingItem] # Backward compatibility: Python >= 3.9 can use ``list[EverythingItem]`` + e: list[EverythingItem] f: str - g: List[int] - h: List[bool] + g: list[int] + h: list[bool] class TestDataclassPayload(TestBase): diff --git a/ipv8/test/messaging/test_serialization.py b/ipv8/test/messaging/test_serialization.py index c26212752..669047102 100644 --- a/ipv8/test/messaging/test_serialization.py +++ b/ipv8/test/messaging/test_serialization.py @@ -1,7 +1,7 @@ from __future__ import annotations import struct -from typing import Any, List, cast +from typing import Any, cast from ...messaging.serialization import DefaultStruct, PackError, Serializable, Serializer from ..base import TestBase @@ -227,10 +227,10 @@ def test_get_formats(self) -> None: for fmt in formats: packer = self.serializer.get_packer_for(fmt) pack_name = f"{packer.__class__.__name__}({fmt})" - self.assertTrue(hasattr(packer, 'pack'), msg='%s has no pack() method' % pack_name) - self.assertTrue(callable(packer.pack), msg='%s.pack is not a method' % pack_name) - self.assertTrue(hasattr(packer, 'unpack'), msg='%s has no unpack() method' % pack_name) - self.assertTrue(callable(packer.unpack), msg='%s.unpack is not a method' % pack_name) + self.assertTrue(hasattr(packer, 'pack'), msg=f'{pack_name} has no pack() method') + self.assertTrue(callable(packer.pack), msg=f'{pack_name}.pack is not a method') + self.assertTrue(hasattr(packer, 'unpack'), msg=f'{pack_name} has no unpack() method') + self.assertTrue(callable(packer.unpack), msg=f'{pack_name}.unpack is not a method') def test_add_packer(self) -> None: """ @@ -278,7 +278,7 @@ def test_serializable_list(self) -> None: instance2 = Short(456) data = self.serializer.pack_serializable_list([instance1, instance2]) - deserialized = cast(List[Short], self.serializer.unpack_serializable_list([Short, Short], data)) + deserialized = cast(list[Short], self.serializer.unpack_serializable_list([Short, Short], data)) self.assertEqual(instance1.number, 123) self.assertEqual(instance1.number, deserialized[0].number) diff --git a/ipv8/test/mocking/endpoint.py b/ipv8/test/mocking/endpoint.py index 2ca398045..6ccab20ca 100644 --- a/ipv8/test/mocking/endpoint.py +++ b/ipv8/test/mocking/endpoint.py @@ -80,12 +80,12 @@ def send(self, socket_address: Address, packet: bytes) -> None: ep = internet[socket_address] get_running_loop().call_soon(ep.notify_listeners, (self.wan_address, packet)) else: - e = AssertionError("Attempted to send data to unregistered address %s" % repr(socket_address)) + e = AssertionError(f"Attempted to send data to unregistered address {socket_address!r}") if self.SEND_INET_EXCEPTION_TO_LOOP: get_running_loop().create_task(crash_event_loop(e)) raise e - def open(self) -> None: # noqa: A003 + def open(self) -> None: """ Set this endpoint to be open. """ diff --git a/ipv8/test/mocking/exit_socket.py b/ipv8/test/mocking/exit_socket.py index eda80e760..80f4844a2 100644 --- a/ipv8/test/mocking/exit_socket.py +++ b/ipv8/test/mocking/exit_socket.py @@ -41,7 +41,8 @@ def sendto(self, data: bytes, destination: Address) -> None: if DataChecker.could_be_bt(data) or DataChecker.could_be_ipv8(data): self.endpoint.send(destination, data) else: - raise AssertionError("Attempted to exit data which is not allowed: %s" % repr(data)) + msg = f"Attempted to exit data which is not allowed: {data!r}" + raise AssertionError(msg) def on_packet(self, packet: tuple[Address, bytes]) -> None: """ diff --git a/ipv8/test/mocking/ipv8.py b/ipv8/test/mocking/ipv8.py index 5a92d8fde..a323b001e 100644 --- a/ipv8/test/mocking/ipv8.py +++ b/ipv8/test/mocking/ipv8.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Awaitable, Generator +from typing import TYPE_CHECKING from ...community import CommunitySettings from ...dht.discovery import DHTDiscoveryCommunity @@ -13,6 +13,8 @@ from .endpoint import AutoMockEndpoint if TYPE_CHECKING: + from collections.abc import Awaitable, Generator + from ...peerdiscovery.discovery import DiscoveryStrategy from ...types import Community, Overlay diff --git a/ipv8/test/test_loader.py b/ipv8/test/test_loader.py index 240f2c428..21d5a1caf 100644 --- a/ipv8/test/test_loader.py +++ b/ipv8/test/test_loader.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Iterable +from typing import TYPE_CHECKING, Any from ..bootstrapping.bootstrapper_interface import Bootstrapper from ..community import Community, CommunitySettings @@ -20,6 +20,8 @@ from .base import TestBase if TYPE_CHECKING: + from collections.abc import Iterable + from ..types import Address, IPv8 diff --git a/ipv8/test/test_peer.py b/ipv8/test/test_peer.py index 878f61a07..447807ee5 100644 --- a/ipv8/test/test_peer.py +++ b/ipv8/test/test_peer.py @@ -118,7 +118,7 @@ def test_to_string(self) -> None: """ Check if the __str__ method functions properly. """ - self.assertEqual(str(self.peer), "Peer<1.2.3.4:5, %s>" % b64encode(self.peer.mid).decode('utf-8')) + self.assertEqual(str(self.peer), "Peer<1.2.3.4:5, {}>".format(b64encode(self.peer.mid).decode('utf-8'))) def test_set_address_init(self) -> None: """ diff --git a/ipv8/types.py b/ipv8/types.py index a573df8f4..b8fbffe76 100644 --- a/ipv8/types.py +++ b/ipv8/types.py @@ -1,6 +1,6 @@ import typing -Address = typing.Tuple[str, int] +Address = tuple[str, int] # ruff: noqa: F401 diff --git a/ipv8/util.py b/ipv8/util.py index 046c6b281..9369c481c 100644 --- a/ipv8/util.py +++ b/ipv8/util.py @@ -4,7 +4,10 @@ import signal import struct from asyncio import Event, Future, iscoroutine -from typing import Any, Awaitable, Callable, Coroutine, TypeVar +from typing import TYPE_CHECKING, Any, Callable, TypeVar + +if TYPE_CHECKING: + from collections.abc import Awaitable, Coroutine maximum_integer = 2147483647 diff --git a/ipv8_service.py b/ipv8_service.py index bdfdf521b..fd24bd399 100644 --- a/ipv8_service.py +++ b/ipv8_service.py @@ -9,7 +9,11 @@ from os.path import isfile from threading import RLock from traceback import format_exception -from typing import Any, Awaitable, Generator +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Awaitable, Generator + from typing import Any if hasattr(sys.modules['__main__'], "IPv8"): sys.modules[__name__] = sys.modules['__main__'] diff --git a/requirements.txt b/requirements.txt index 2addbcc69..275a81d28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,9 @@ cryptography libnacl -aiohttp==3.8.6; python_version=='3.7' -aiohttp>=3.9.1; python_version>'3.7' +aiohttp aiohttp_apispec>=3.0.0b1 pyOpenSSL pyasn1 -asynctest; python_version=='3.7' marshmallow typing-extensions packaging diff --git a/run_all_tests.py b/run_all_tests.py index d59445996..c48207b25 100644 --- a/run_all_tests.py +++ b/run_all_tests.py @@ -14,11 +14,12 @@ import time import unittest from concurrent.futures import ProcessPoolExecutor -from typing import TYPE_CHECKING, Generator, TextIO, cast +from typing import TYPE_CHECKING, TextIO, cast from unittest import TestCase if TYPE_CHECKING: import types + from collections.abc import Generator from typing_extensions import Self @@ -218,8 +219,7 @@ def task_test(*test_names: str) -> tuple[bool, int, float, list[tuple[str, str, return tests_failed, tests_run_count, end_time - start_time, combined_event_log, print_stream.getvalue() -def scan_for_test_files(directory: pathlib.Path | str = pathlib.Path('./ipv8/test')) -> Generator[pathlib.Path, - None, None]: +def scan_for_test_files(directory: pathlib.Path | str = pathlib.Path('./ipv8/test')) -> Generator[pathlib.Path]: """ Find Python files starting with ``test_`` in a given directory. """ diff --git a/scripts/tracker_service.py b/scripts/tracker_service.py index 4fa63f242..4e8e0db00 100644 --- a/scripts/tracker_service.py +++ b/scripts/tracker_service.py @@ -44,7 +44,7 @@ class TrackerChurn(RandomChurn): Strategy to get rid of unresponsive of superfluous peers. """ - def __init__(self, overlay: Overlay, sample_size: int = 8, # noqa: PLR0913 + def __init__(self, overlay: Overlay, sample_size: int = 8, ping_interval: float = 10.0, inactive_time: float = 27.5, drop_time: float = 57.5, max_peers: int = 100) -> None: """ diff --git a/setup.py b/setup.py index 6c58d3d78..395409526 100644 --- a/setup.py +++ b/setup.py @@ -26,8 +26,8 @@ "packaging" ], extras_require={ - "all": ["asynctest; python_version=='3.7'", "coverage"], - "tests": ["asynctest; python_version=='3.7'", "coverage"] + "all": ["coverage"], + "tests": ["coverage"] }, classifiers=[ "Development Status :: 5 - Production/Stable", @@ -35,12 +35,12 @@ "License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)", "Natural Language :: English", "Operating System :: OS Independent", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Scientific/Engineering", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Distributed Computing", diff --git a/stresstest/bootstrap_introductions.py b/stresstest/bootstrap_introductions.py index ced125c04..15a9ce8fe 100644 --- a/stresstest/bootstrap_introductions.py +++ b/stresstest/bootstrap_introductions.py @@ -136,7 +136,7 @@ async def main() -> None: await event.wait() await ipv8.stop() - with open('bootstrap_introductions.txt', 'w') as f: # noqa: ASYNC101 + with open('bootstrap_introductions.txt', 'w') as f: # noqa: ASYNC230 f.write('Address Peers Type') for dns_addr, responses in overlay.introductions.items(): f.write(f"\n{dns_addr[0]}:{dns_addr[1]} {len([wan for wan, _, _ in responses if wan != ('0.0.0.0', 0)])} 0") diff --git a/stresstest/peer_discovery_defaults.py b/stresstest/peer_discovery_defaults.py index 9e394387e..d9ffab412 100644 --- a/stresstest/peer_discovery_defaults.py +++ b/stresstest/peer_discovery_defaults.py @@ -96,4 +96,4 @@ async def start_communities() -> None: run(start_communities()) # Print the introduction times for all default Communities, sorted alphabetically. -print(','.join(['%.4f' % RESULTS[key] for key in sorted(RESULTS)])) # noqa: T201 +print(','.join([f'{RESULTS[key]:.4f}' for key in sorted(RESULTS)])) # noqa: T201