Merge pull request #682 from valory-xyz/feat/port-http-server

Port HTTP server as valory package

Author: angrybayblade

HISTORY.md

@@ -1,6 +1,22 @@
 # Release History - open AEA
 
 
+## 1.40.0 (2023-09-26)
+
+AEA:
+- Fixes the source repository validation regex
+- Updates the `generate-key` command to prompt before overwriting the existing keys file
+- Fixes the inconsistent hashing caused by the `CRLF` line endings
+- Updates the component loader to ignore the test modules when loading the component
+- Adds support for overriding dependencies
+- Updates the `sync` command to download missing dependencies and update `packages.json`
+- Updates the error messages for missing ledger plugins on `generate-key` command
+  
+Plugins:
+- Adds missing `py.typed` markers
+- Backports the changes from the `agent-academy-2` repository on the ledger connection
+- Ports `http_server` as a valory connection
+
 ## 1.40.0 (2023-09-26)
 
 AEA:

SECURITY.md

@@ -8,8 +8,8 @@ The following table shows which versions of `open-aea` are currently being suppo
 
 | Version   | Supported          |
 | --------- | ------------------ |
-| `1.40.x`   | :white_check_mark: |
-| `< 1.40.0` | :x:                |
+| `1.41.x`   | :white_check_mark: |
+| `< 1.41.0` | :x:                |
 
 ## Reporting a Vulnerability
 

aea/__version__.py

@@ -23,7 +23,7 @@
 __title__ = "open-aea"
 __description__ = "Open Autonomous Economic Agent framework (without vendor lock-in)"
 __url__ = "https://github.com/valory-xyz/open-aea.git"
-__version__ = "1.40.0"
+__version__ = "1.41.0"
 __author__ = "Valory AG"
 __license__ = "Apache-2.0"
 __copyright__ = "2021 Valory AG, 2019 Fetch.AI Limited"

aea/helpers/ipfs/base.py

@@ -22,12 +22,15 @@
 import hashlib
 import io
 import os
+import platform
+import re
 from pathlib import Path
 from typing import Any, Dict, Generator, Optional, Sized, Tuple, cast
 
 import base58
 
 from aea.helpers.cid import to_v1
+from aea.helpers.io import open_file
 from aea.helpers.ipfs.utils import _protobuf_python_implementation
 
 
@@ -41,12 +44,34 @@
     from aea.helpers.ipfs.pb.merkledag_pb2 import PBNode  # type: ignore
 
 
+def _is_text(file_path: str) -> bool:
+    """Check if a file can be read as text or not."""
+    try:
+        with open_file(file_path, "r") as f:
+            f.readline()
+        return True
+    except UnicodeDecodeError:
+        return False
+
+
+def _dos2unix(file_content: bytes) -> bytes:
+    """
+    Replace occurrences of Windows line terminator CR/LF with only LF.
+
+    :param file_content: the content of the file.
+    :return: the same content but with the line terminator
+    """
+    return re.sub(b"\r\n", b"\n", file_content, flags=re.M)
+
+
 def _read(file_path: str) -> bytes:
     """Read and verify the file is not empty."""
     with open(file_path, "rb") as file:
         data = file.read()
     if len(data) == 0:
         raise ValueError(f"File cannot be empty: {file_path}")
+    if platform.system() == "Windows" and _is_text(file_path=file_path):
+        data = _dos2unix(data)
     return data
 
 

deploy-image/Dockerfile

@@ -16,7 +16,7 @@ RUN apk add --no-cache go
 
 # aea installation
 RUN pip install --upgrade pip
-RUN pip install --upgrade --force-reinstall open-aea[all]==1.40.0 "open-aea-cli-ipfs<2.0.0,>=1.40.0"
+RUN pip install --upgrade --force-reinstall open-aea[all]==1.41.0 "open-aea-cli-ipfs<2.0.0,>=1.41.0"
 
 # directories and aea cli config
 WORKDIR /home/agents

deploy-image/README.md

@@ -11,7 +11,7 @@ The example uses the `fetchai/my_first_aea` project. You will likely want to mod
 Install subversion, then download the example directory to your local working directory
 
 ``` bash
-svn checkout https://github.com/valory-xyz/open-aea/tags/v1.40.0/packages packages
+svn checkout https://github.com/valory-xyz/open-aea/tags/v1.41.0/packages packages
 ```
 
 ### Modify scripts

develop-image/docker-env.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Swap the following lines if you want to work with 'latest'
-DOCKER_IMAGE_TAG=valory/open-aea-develop:1.40.0
+DOCKER_IMAGE_TAG=valory/open-aea-develop:1.41.0
 # DOCKER_IMAGE_TAG=valory/open-aea-develop:latest
 
 DOCKER_BUILD_CONTEXT_DIR=..

docs/aev-echo-demo.md

@@ -23,7 +23,7 @@ aea_version: '>=1.3.0, <2.0.0'
 fingerprint: {}
 fingerprint_ignore_patterns: []
 connections:
-- fetchai/http_server:0.22.0
+- valory/http_server:0.22.0
 contracts: []
 protocols:
 - fetchai/default:1.0.0
@@ -45,15 +45,15 @@ dependencies:
   open-aea-ledger-ethereum: {}
 default_connection: null
 ---
-public_id: fetchai/http_server:0.22.0
+public_id: valory/http_server:0.22.0
 type: connection
 config:
   host: ${HOST:str:localhost}
   port: ${PORT:int:5000}
   target_skill_id: ${TARGET_SKILL:str:fetchai/http_echo:0.20.0}
 ```
 
-Notice how the ```fetchai/http_server:0.22.0``` has a number of override parameters specified:
+Notice how the ```valory/http_server:0.22.0``` has a number of override parameters specified:
 ``` yaml
 host: ${HOST:str:localhost}
 port: ${PORT:int:5000}

docs/connect-a-frontend.md

@@ -3,7 +3,7 @@ This page lays out two options for connecting a front-end to an AEA. The followi
 <img src="../assets/http-integration.svg" alt="How to connect front-end to your AEA" class="center" style="display: block; margin-left: auto; margin-right: auto;width:80%;">
 
 ## Case 1
-The first option is to create a `HTTP Server` connection that handles incoming requests from a REST API. In this scenario, the REST API communicates with the AEA and requests are handled by the `HTTP Server` connection package. The REST API should send CRUD requests to the `HTTP Server` connection (`fetchai/http_server:0.22.0`) which translates these into Envelopes to be consumed by the correct skill.
+The first option is to create a `HTTP Server` connection that handles incoming requests from a REST API. In this scenario, the REST API communicates with the AEA and requests are handled by the `HTTP Server` connection package. The REST API should send CRUD requests to the `HTTP Server` connection (`valory/http_server:0.22.0`) which translates these into Envelopes to be consumed by the correct skill.
 
 ## Case 2
 The second option is to create a front-end comprising a stand-alone `Multiplexer` with a `P2P` connection (`fetchai/p2p_libp2p:0.25.0`). In this scenario the <a href="../acn">Agent Communication Network</a> can be used to send Envelopes from the AEA to the front-end.

docs/connection.md

@@ -36,7 +36,7 @@ The developer needs to implement four public coroutines:
 
 - The `receive` coroutine is continuously called by the AEA framework. It either returns `None` or an envelope. The `receive` coroutine must implement the logic of data being received by the agent, and if necessary, its translation into a relevant protocol.
 
-The framework provides a demo `stub` connection which implements an I/O reader and writer to send and receive messages between the agent and a local file. To gain inspiration and become familiar with the structure of connection packages, you may find it useful to check out `fetchai/stub:0.21.0`, `fetchai/http_server:0.22.0` or `valory/http_client:0.23.0` connections. The latter two connections are for external clients to connect with an agent, and for the agent to connect with external servers, respectively.
+The framework provides a demo `stub` connection which implements an I/O reader and writer to send and receive messages between the agent and a local file. To gain inspiration and become familiar with the structure of connection packages, you may find it useful to check out `fetchai/stub:0.21.0`, `valory/http_server:0.22.0` or `valory/http_client:0.23.0` connections. The latter two connections are for external clients to connect with an agent, and for the agent to connect with external servers, respectively.
 
 ### Primary methods to develop - sync connection interface
 

docs/http-connection-and-skill.md

@@ -29,8 +29,8 @@ Add the http server connection package:
 mkdir packages
 aea create my_aea
 cd my_aea
-aea add connection fetchai/http_server:0.22.0:bafybeihvscddpxjbtqsetngmxo3kiht2wqhosmwiyuh3f6zjti3x3byu5u --remote
-aea push connection fetchai/http_server --local
+aea add connection valory/http_server:0.22.0:bafybeie3d6tj2kzewced4nnttxdvr7njue2ortvawmwdt67gwf4tcuosom --remote
+aea push connection valory/http_server --local
 aea add protocol fetchai/default:1.0.0:bafybeibtqp56jkijwjsohk4z5vqp6pfkiexmnmk5uleteotbsgrypy6gxm --remote
 aea push protocol fetchai/default --local
 aea add protocol valory/http:1.0.0:bafybeiejoqgv7finfxo3rcvvovrlj5ccrbgxodjq43uo26ylpowsa3llfe --remote
@@ -42,7 +42,7 @@ aea delete my_aea
 Update the default connection:
 
 ``` bash
-aea config set agent.default_connection fetchai/http_server:0.22.0
+aea config set agent.default_connection valory/http_server:0.22.0
 ```
 
 Modify the `api_spec_path`:

docs/http-echo-demo.md

@@ -16,7 +16,7 @@ The easiest way to get started with the http server is to use our pre-built exam
 
 ``` bash
 pipenv shell
-aea fetch open_aea/http_echo:0.1.0:bafybeie6dxx5zbu6bz7at7kqj3thjvvzucj45f3tksbio5dg23ywsbb2ty --remote
+aea fetch open_aea/http_echo:0.1.0:bafybeiaosh5h4msjh4rubd3xnygpqco47hjv4zbt6cz4zrai6zmvd77bv4 --remote
 cd http_echo
 aea generate-key ethereum; aea add-key ethereum
 aea install
@@ -27,8 +27,8 @@ Adding protocol 'valory/http:1.0.0'...
 Successfully added protocol 'valory/http:1.0.0'.
 Adding protocol 'fetchai/default:1.0.0'...
 Successfully added protocol 'fetchai/default:1.0.0'.
-Adding connection 'fetchai/http_server:0.22.0'...
-Successfully added connection 'fetchai/http_server:0.22.0'.
+Adding connection 'valory/http_server:0.22.0'...
+Successfully added connection 'valory/http_server:0.22.0'.
 Adding skill 'fetchai/http_echo:0.20.0'...
 Successfully added skill 'fetchai/http_echo:0.20.0'.
 Agent http_echo successfully fetched.

docs/interaction-protocol.md

@@ -18,7 +18,7 @@ Usually, an interaction involves three types of framework packages: <a href="../
 
 ### Example 1: AEA <> web client
 
-In the <a href="../http-connection-and-skill">http connection guide</a> we demonstrate how an AEA with an http server connection (e.g. `fetchai/http_server`) receives http payloads from web clients, translates them to messages conforming with the `fetchai/http` protocol and passes it to a skill (e.g. `fetchai/http_echo`) to process. The `fetchai/http` protocol in this case is used for communication between the connection and the skill.
+In the <a href="../http-connection-and-skill">http connection guide</a> we demonstrate how an AEA with an http server connection (e.g. `valory/http_server`) receives http payloads from web clients, translates them to messages conforming with the `fetchai/http` protocol and passes it to a skill (e.g. `fetchai/http_echo`) to process. The `fetchai/http` protocol in this case is used for communication between the connection and the skill.
 
 ### Example 2 : AEA <> 3rd party server
 

docs/upgrading.md

@@ -9,7 +9,7 @@ Below we describe the additional manual steps required to upgrade between differ
 
 ### Upgrade guide
 
-## `v1.40.0` to `v1.40.1`
+## `v1.40.0` to `v1.41.0`
 
 - The way the dependencies will be selected for installation when running `aea install` has changed. Before this version, the versions were being merging all of the versions for a python package and using the most compatible version specifier possible. With this release, this behaviour will be replaced by overriding the dependencies in the following order `extra dependencies provided by flag > agent > skill > connection > contract > protocol` what this means is, let's say you have 3 packages with a same python package as a dependency
 

examples/tac_deploy/Dockerfile

@@ -19,7 +19,7 @@ RUN apk add --no-cache go
 
 # aea installation
 RUN python -m pip install --upgrade pip
-RUN pip install --upgrade --force-reinstall open-aea[all]==1.40.0
+RUN pip install --upgrade --force-reinstall open-aea[all]==1.41.0
 
 # directories and aea cli config
 COPY /.aea /home/.aea

packages/open_aea/agents/http_echo/aea-config.yaml

@@ -8,7 +8,7 @@ fingerprint:
   README.md: bafybeibkr6ecv5efx3hwxvxposvpmr76ugrj6kydeasb7bppo3ibynnjcu
 fingerprint_ignore_patterns: []
 connections:
-- fetchai/http_server:0.22.0:bafybeihvscddpxjbtqsetngmxo3kiht2wqhosmwiyuh3f6zjti3x3byu5u
+- valory/http_server:0.22.0:bafybeie3d6tj2kzewced4nnttxdvr7njue2ortvawmwdt67gwf4tcuosom
 contracts: []
 protocols:
 - fetchai/default:1.0.0:bafybeibtqp56jkijwjsohk4z5vqp6pfkiexmnmk5uleteotbsgrypy6gxm
@@ -29,7 +29,7 @@ dependencies:
   open-aea-ledger-ethereum: {}
 default_connection: null
 ---
-public_id: fetchai/http_server:0.22.0
+public_id: valory/http_server:0.22.0
 type: connection
 config:
   host: ${HOST:str:localhost}

packages/packages.json

@@ -7,9 +7,9 @@
         "protocol/valory/contract_api/1.0.0": "bafybeialhbjvwiwcnqq3ysxcyemobcbie7xza66gaofcvla5njezkvhcka",
         "protocol/valory/http/1.0.0": "bafybeiejoqgv7finfxo3rcvvovrlj5ccrbgxodjq43uo26ylpowsa3llfe",
         "protocol/valory/ledger_api/1.0.0": "bafybeige5agrztgzfevyglf7mb4o7pzfttmq4f6zi765y4g2zvftbyowru",
-        "connection/fetchai/http_server/0.22.0": "bafybeihvscddpxjbtqsetngmxo3kiht2wqhosmwiyuh3f6zjti3x3byu5u",
         "connection/fetchai/stub/0.21.0": "bafybeictgpdqbpyppmoxn2g7jkaxvulihew7zaszv4xyhgvsntq7tqs7wi",
         "connection/valory/ledger/0.19.0": "bafybeigo5vst3zlltkouenwxuzn6c47yr2fbbml6dl2o32rfnsezmalgnu",
+        "connection/valory/http_server/0.22.0": "bafybeie3d6tj2kzewced4nnttxdvr7njue2ortvawmwdt67gwf4tcuosom",
         "connection/valory/p2p_libp2p/0.1.0": "bafybeiaykya7tvir7k5scovjzuagpfcftvptxoi2od5qqqvukwglsrrtzy",
         "connection/valory/p2p_libp2p_client/0.1.0": "bafybeihge56dn3xep2dzomu7rtvbgo4uc2qqh7ljl3fubqdi2lq44gs5lq",
         "connection/valory/p2p_libp2p_mailbox/0.1.0": "bafybeieufv6tbei3vza7bg7ggzsvyvwmzps5kghxlar3drhme6d7mxawvy",
@@ -26,7 +26,7 @@
         "agent/fetchai/gym_aea/0.25.0": "bafybeif5c657r3ixuosbyihtsdzr2tqifikxgy5e6t7ur2jmhtiolina3m",
         "agent/fetchai/my_first_aea/0.27.0": "bafybeicrgya4gln4lqr3ba76t2cuyik37lcgr64kzna2fkoseseobytaa4",
         "agent/open_aea/gym_aea/0.1.0": "bafybeibyyl4wtnrfxeoed3kjnlftphkcikdqrgf76u34v4wbvwgsqfqrgy",
-        "agent/open_aea/http_echo/0.1.0": "bafybeie6dxx5zbu6bz7at7kqj3thjvvzucj45f3tksbio5dg23ywsbb2ty",
+        "agent/open_aea/http_echo/0.1.0": "bafybeiaosh5h4msjh4rubd3xnygpqco47hjv4zbt6cz4zrai6zmvd77bv4",
         "agent/open_aea/my_first_aea/0.1.0": "bafybeibv7nlyxldyj5ntivsu74ylul4dltpfvkfa46k2pbveetfpkvz4jm",
         "connection/fetchai/local/0.20.0": "bafybeigukufpgy2sdkv5qqde47dhigbjitxcrxwg6sl6unlpbc7wbxqyhm",
         "connection/valory/http_client/0.23.0": "bafybeifgeqgryx6b3s6eseyzyezygmeitcpt3tkor2eiycozoi6clgdrny",

packages/fetchai/connections/http_server/README.md packages/valory/connections/http_server/README.md

@@ -4,4 +4,4 @@ This connection wraps an HTTP server. It consumes requests from clients, transla
 
 ## Usage
 
-First, add the connection to your AEA project (`aea add connection fetchai/http_server:0.22.0`). Then, update the `config` in `connection.yaml` by providing a `host` and `port` of the server. Optionally, provide a path to an [OpenAPI specification](https://swagger.io/docs/specification/about/) for request validation.
+First, add the connection to your AEA project (`aea add connection valory/http_server:0.22.0`). Then, update the `config` in `connection.yaml` by providing a `host` and `port` of the server. Optionally, provide a path to an [OpenAPI specification](https://swagger.io/docs/specification/about/) for request validation.

packages/fetchai/connections/http_server/__init__.py packages/valory/connections/http_server/__init__.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2020 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");

packages/fetchai/connections/http_server/connection.py packages/valory/connections/http_server/connection.py

@@ -67,7 +67,7 @@
 _default_logger = logging.getLogger("aea.packages.fetchai.connections.http_server")
 
 RequestId = DialogueLabel
-PUBLIC_ID = PublicId.from_str("fetchai/http_server:0.22.0")
+PUBLIC_ID = PublicId.from_str("valory/http_server:0.22.0")
 
 
 class HttpDialogues(BaseHttpDialogues):

packages/fetchai/connections/http_server/connection.yaml packages/valory/connections/http_server/connection.yaml

@@ -1,21 +1,21 @@
 name: http_server
-author: fetchai
+author: valory
 version: 0.22.0
 type: connection
 description: The HTTP server connection that wraps http server implementing a RESTful
   API specification.
 license: Apache-2.0
 aea_version: '>=1.0.0, <2.0.0'
 fingerprint:
-  README.md: bafybeihkuhhsdfw5qqtz2jwpfppub6yvsehzmvmaqjlxnal4v76x47mcrq
-  __init__.py: bafybeif5pkr5oarwd7yagdgn46miolmdmvgdyxv4kadgws2bf3iwshom24
-  connection.py: bafybeicrzlp4krqcf6gunbhhu2j47qp4f7adviw3qxoabt56yy5d3ksaci
-  tests/__init__.py: bafybeidpthyhnhsvpixejrud77klcfkb3titlmyrssovfzt4a76dn3wnpm
+  README.md: bafybeifepluovb4so2eem34ulpcp4svegb2dqfpmtojtofhfkb3j2xanui
+  __init__.py: bafybeifykou5sazojmc7hdqnsdp4mncd4zh3xys3mdgdzwks23mvhzu2ga
+  connection.py: bafybeidc6zysa47tppkvq3ddq4jgpind7h6zzpsl4s7vukbbj5y4ts2zxm
+  tests/__init__.py: bafybeifqaf7cnc4oczjkbwmv4ahrkbiqxrojwgowej3kbri3skz4lzt43i
   tests/data/certs/server.crt: bafybeiev5i3xxkvn36wflf633gkumuxexsw4y2bubwbvl7edrz4igfgv34
   tests/data/certs/server.csr: bafybeicvp7xdl5w3o4bzikkudpduitss3bpp6xqfwlxbw6kabdangohy5u
   tests/data/certs/server.key: bafybeiabvpkpqr4fctrbssfal6pviv5otgmu32qyrfpyhcql5wgmlzjtoe
   tests/data/petstore_sim.yaml: bafybeiaekkfxljlv57uviz4ug6isdqbzsnuxpsgy3dvhzh22daql3xh2i4
-  tests/test_http_server.py: bafybeihxrl4eqgbroqfcrnued3twwx24cfzfpuxrqkkgvirqtqhadbmxzm
+  tests/test_http_server.py: bafybeia2cax2paqaygr6xgzbgmiqepsyjkdx4joi5qukpg6ewnz5yckk7m
 fingerprint_ignore_patterns: []
 connections: []
 protocols:

packages/fetchai/connections/http_server/tests/__init__.py packages/valory/connections/http_server/tests/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2020 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");

packages/fetchai/connections/http_server/tests/data/certs/server.crt packages/valory/connections/http_server/tests/data/certs/server.crt

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # ------------------------------------------------------------------------------
 #
-#   Copyright 2022 Valory AG
+#   Copyright 2022-2023 Valory AG
 #   Copyright 2018-2021 Fetch.AI Limited
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
@@ -41,7 +41,7 @@
 from aea.test_tools.mocks import RegexComparator
 from aea.test_tools.network import get_host, get_unused_tcp_port
 
-from packages.fetchai.connections.http_server.connection import (
+from packages.valory.connections.http_server.connection import (
     APISpec,
     HTTPServerConnection,
     Response,

packages/fetchai/connections/http_server/tests/data/certs/server.csr packages/valory/connections/http_server/tests/data/certs/server.csr

@@ -26,7 +26,7 @@
 
 setup(
     name="open-aea-cli-benchmark",
-    version="1.40.0",
+    version="1.41.0",
     author="Valory AG",
     license="Apache-2.0",
     description="CLI extension for AEA framework benchmarking.",

packages/fetchai/connections/http_server/tests/data/certs/server.key packages/valory/connections/http_server/tests/data/certs/server.key

@@ -28,7 +28,7 @@
 
 setup(
     name="open-aea-cli-ipfs",
-    version="1.40.0",
+    version="1.41.0",
     author="Valory AG",
     license="Apache-2.0",
     description="CLI extension for open AEA framework wrapping IPFS functionality.",

packages/fetchai/connections/http_server/tests/data/petstore_sim.yaml packages/valory/connections/http_server/tests/data/petstore_sim.yaml

@@ -26,7 +26,7 @@
 
 setup(
     name="open-aea-ledger-cosmos",
-    version="1.40.0",
+    version="1.41.0",
     author="Valory AG",
     license="Apache-2.0",
     description="Python package wrapping the public and private key cryptography and ledger api of Cosmos.",

packages/fetchai/connections/http_server/tests/test_http_server.py packages/valory/connections/http_server/tests/test_http_server.py

@@ -25,7 +25,7 @@
 
 setup(
     name="open-aea-ledger-ethereum-flashbots",
-    version="1.40.0",
+    version="1.41.0",
     author="Valory AG",
     license="Apache-2.0",
     description="Python package extending the default open-aea ethereum ledger plugin to add support for flashbots.",
@@ -41,7 +41,7 @@
     },
     python_requires=">=3.9,<4.0",
     install_requires=[
-        "open-aea-ledger-ethereum~=1.40.0",
+        "open-aea-ledger-ethereum~=1.41.0",
         "open-aea-flashbots==1.4.0",
     ],
     tests_require=["pytest"],