[SDESK-7289] Feature: HTTP Endpoints (#2637)

* fix issues from flask upgrade

* chore: Update requirements

* Fix lint issues

* Feature: http endpoints

* Add/update tests

* Update docs

* Add flask to mypy-requirements.txt file

* Get raw data not model instance for http endpoint

* Improve flask route abstraction

* Fix async test client

* fix quart version

* Provide both delete and delete_many in resource service

* Rename ResourceModelConfig to just ResourceConfig

* Move Resource REST config to ResourceConfig
Also rename http module to web, and remove HTTP prefix from classes

* Run new core unit tests separately

* fix: Mongo index options sending None

* fix async test client with asgiref lib sending empty body

* use flask.Blueprint when registering EndpointGroup to wsgi

.github/workflows/nose-tests.yml

@@ -19,8 +19,8 @@ jobs:
     - run: ./scripts/tests_setup
     - run: pip install -U pip wheel setuptools
     - run: pip install -r dev-requirements.txt
-
-    - run: pytest --log-level=ERROR --disable-warnings
+    - run: pytest --log-level=ERROR --disable-warnings tests/core
+    - run: pytest --log-level=ERROR --disable-warnings --ignore=tests/core
 
   pip-compile:
     runs-on: ubuntu-latest

docs/core/app.rst

@@ -23,8 +23,3 @@ APP References
 .. autoclass:: superdesk.core.app.SuperdeskAsyncApp
     :member-order: bysource
     :members:
-
-.. autoclass:: superdesk.core.wsgi.WSGIApp
-    :member-order: bysource
-    :members:
-    :undoc-members:

docs/core/resource_management.rst

@@ -3,12 +3,10 @@
 Resource Management
 ===================
 
-.. module:: superdesk.core.resources
-
 Resource Services:
 ------------------
 
-The management of resources is performed using the :class:`AsyncResourceService <service.AsyncResourceService>` class
+The management of resources is performed using the :class:`AsyncResourceService <superdesk.core.resources.service.AsyncResourceService>` class
 instances. This is similar to how it is done in Superdesk < v3.0, with some slight improvements.
 
 One major difference is the need to directly import the resource service from the module, and not use

docs/core/resources.rst

@@ -341,7 +341,7 @@ data type that has a different mapping than the default, you can inherit from th
 Registering Resources
 ---------------------
 The :meth:`Resources.register <model.Resources.register>` method provides a way to register a resource with the system,
-using the :class:`ResourceModelConfig <model.ResourceModelConfig>` class to provide the resource config.
+using the :class:`ResourceConfig <model.ResourceConfig>` class to provide the resource config.
 
 This will register the resource with MongoDB and optionally the Elasticsearch system. See
 :class:`MongoResourceConfig <superdesk.core.mongo.MongoResourceConfig>` and
@@ -355,14 +355,12 @@ Example module::
     from superdesk.core.module import Module, SuperdeskAsyncApp
     from superdesk.core.resources import (
         ResourceModel,
-        ResourceModelConfig,
+        ResourceConfig,
         fields,
-    )
-    from superdesk.core.mongo import (
         MongoResourceConfig,
         MongoIndexOptions,
+        ElasticResourceConfig,
     )
-    from superdesk.core.elastic.resources import ElasticResourceConfig
 
     # Define your user model
     class User(ResourceModel):
@@ -373,7 +371,7 @@ Example module::
         code: Optional[fields.Keyword] = None
 
     # Define the resource config
-    user_model_config = ResourceModelConfig(
+    user_model_config = ResourceConfig(
         name="users",
         data_class=User,
         mongo=[
@@ -418,7 +416,7 @@ Resource Model
     :member-order: bysource
     :members: id
 
-.. autoclass:: superdesk.core.resources.model.ResourceModelConfig
+.. autoclass:: superdesk.core.resources.model.ResourceConfig
     :member-order: bysource
     :members:
 

docs/core/web.rst

@@ -0,0 +1,251 @@
+.. core_http:
+
+Web Server
+==========
+
+.. module:: superdesk.core.web
+
+In Superdesk v3.0+ we use an abstraction layer when it comes to the web server functionality. This means that
+we don't directly interact with the Flask library, but Superdesk specific classes and functionality instead.
+
+For example::
+
+    from superdesk.core.module import Module
+    from superdesk.core.web import (
+        endpoint,
+        Request,
+        Response,
+    )
+
+    @endpoint("hello/world", methods=["GET"])
+    async def hello_world(request: Request) -> Response:
+        return Response(
+            body="hello, world!",
+            status_code=200,
+            headers=()
+        )
+
+    module = Module(
+        name="tests.http",
+        endpoints=[hello_world]
+    )
+
+
+Declaring Endpoints
+-------------------
+
+There are three ways to declare a function as an HTTP endpoint.
+
+1. Using the :func:`endpoint <types.endpoint>` decorator::
+
+    from superdesk.core.web import (
+        Request,
+        Response,
+        endpoint,
+    )
+
+    @endpoint("hello/world", methods=["GET"])
+    async def hello_world(request: Request) -> Response:
+        ...
+
+2. Using the :class:`Endpoint <types.Endpoint>` class::
+
+    from superdesk.core.web import (
+        Request,
+        Response,
+        Endpoint,
+    )
+
+    async def hello_world(request: Request) -> Response:
+        ...
+
+    endpoint = Endpoint(
+        url="hello/world",
+        methods=["GET"],
+        func=hello_world
+    )
+
+3. Using the :class:`EndpointGroup <types.EndpointGroup>` class group::
+
+    from superdesk.core.web import (
+        Request,
+        Response,
+        EndpointGroup,
+    )
+
+    group = EndpointGroup(
+        name="tests",
+        import_name=__name__,
+        url_prefix="hello"
+    )
+
+    @group.endpoint(url="world", methods=["GET"])
+    async def hello_world(request: Request) -> Response:
+        ...
+
+Registering Endpoints:
+----------------------
+
+There are two ways to register an endpoint with the system.
+
+1. Manually using the :attr:`WSGIApp.register_endpoint <types.WSGIApp.register_endpoint>` function::
+
+    from superdesk.core.app import SuperdeskAsyncApp
+
+    def init(app: SuperdeskAsyncApp):
+        app.wsgi.register_endpoint(hello_world)
+
+2. Automatically with the :attr:`endpoints <superdesk.core.module.Module.endpoints>` module config::
+
+    from superdesk.core.module import Module
+
+    module = Module(
+        name="my.module",
+        endpoints=[hello_world, group]
+    )
+
+Resource REST Endpoints
+-----------------------
+
+REST endpoints can be enabled for a resource by defining the
+:attr:`rest_endpoints <superdesk.core.resources.model.ResourceConfig.rest_endpoints>` attribute on the ResourceConfig.
+See :class:`RestEndpointConfig <superdesk.core.resources.resource_rest_endpoints.RestEndpointConfig>` for config
+options.
+
+For example::
+
+    from superdesk.core.module import Module
+    from superdesk.core.resources import (
+        ResourceConfig,
+        ResourceModel,
+        RestEndpointConfig,
+    )
+
+    # Define your resource model and config
+    class User(ResourceModel):
+        first_name: str
+        last_name: str
+
+    # Configure the resource
+    user_resource_config = ResourceConfig(
+        name="users",
+        data_class=User,
+
+        # Including the `rest_endpoints` config
+        rest_endpoints=RestEndpointConfig(),
+    )
+
+    module = Module(name="tests.users")
+
+
+Validation
+----------
+
+Request route arguments and URL params can be validated against Pydantic models. All you need to do is to define
+the model for each argument type, and the system will validate them when processing the request.
+If the request does not pass validation, a Pydantic ValidationError will be raised.
+
+For example::
+
+    from typing import Optional
+    from enum import Enum
+    from pydantic import BaseModel
+
+    class UserActions(str, Enum):
+        activate = "activate"
+        disable = "disable"
+
+    class RouteArguments(BaseModel):
+        user_id: str
+        action: UserActions
+
+    class URLParams(BaseModel):
+        verbose: bool = False
+
+    @endpoint(
+        "users_async/<string:user_id>/action/<string:action>",
+        methods=["GET"]
+    )
+    async def hello_world(
+        args: RouteArguments,
+        params: URLParams,
+        request: Request,
+    ) -> Response:
+        # If the request reaches this line,
+        # we have valid arguments & params
+        user = get_user(args.user_id)
+        if args.action == UserActions.activate:
+            pass
+        elif args.action == UserActions.disable:
+            pass
+        else:
+            # This line should never be reached,
+            # as validation would have caught this already
+            assert False, "unreachable"
+
+        return Response(
+            "hello, world!",
+            200,
+            ()
+        )
+
+    def init(app: SuperdeskAsyncApp) -> None:
+        app.wsgi.register_endpoint(hello_world)
+
+API References
+--------------
+
+.. autoclass:: superdesk.core.web.types.Response
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autodata:: superdesk.core.web.types.EndpointFunction
+
+.. autoclass:: superdesk.core.web.types.Endpoint
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autoclass:: superdesk.core.web.types.Request
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autoclass:: superdesk.core.web.types.EndpointGroup
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autoclass:: superdesk.core.web.types.RestResponseMeta
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autoclass:: superdesk.core.web.types.RestGetResponse
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autofunction:: superdesk.core.web.types.endpoint
+
+.. autoclass:: superdesk.core.web.rest_endpoints.RestEndpoints
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+
+.. autoclass:: superdesk.core.resources.resource_rest_endpoints.RestEndpointConfig
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autoclass:: superdesk.core.resources.resource_rest_endpoints.ResourceRestEndpoints
+    :member-order: bysource
+    :members:
+    :undoc-members:
+
+.. autoclass:: superdesk.core.web.types.WSGIApp
+    :member-order: bysource
+    :members:
+    :undoc-members:

docs/index.rst

@@ -47,6 +47,7 @@ existing code to the new framework.
    core/modules
    core/resources
    core/resource_management
+   core/web
    core/mongo
    core/elastic
 

mypy-requirements.txt

@@ -20,3 +20,4 @@ boto3-stubs[s3,sqs]
 motor-types
 pydantic
 flask[async]
+quart>=0.19.6,<0.20.0

setup.py

@@ -18,6 +18,7 @@
     "eve-elastic>=7.4.0,<7.5.0",
     "elasticsearch[async]<7.18",  # we are using oss version on test server
     "flask[async]>=3.0",
+    "quart>=0.19.6,<0.20.0",
     "flask-mail>=0.9,<0.11",
     "flask-babel>=1.0,<4.1",
     "arrow>=0.4,<=1.3.0",

superdesk/core/app.py

@@ -11,7 +11,7 @@
 from typing import Dict, List, Optional
 import importlib
 
-from .wsgi import WSGIApp
+from .web import WSGIApp
 
 
 class SuperdeskAsyncApp:
@@ -81,6 +81,25 @@ def _load_modules(self, paths: List[str]):
             for resource_config in module.resources or []:
                 self.resources.register(resource_config)
 
+        # Now register all http endpoints
+        for module in self.get_module_list():
+            from .resources.resource_rest_endpoints import ResourceRestEndpoints
+
+            if module.endpoints is None:
+                module.endpoints = []
+            for resource_config in module.resources or []:
+                # If REST endpoints are enabled for this resource
+                # then add the endpoint group to this module's `endpoints` config
+                rest_endpoint_config = resource_config.rest_endpoints
+                if rest_endpoint_config is None:
+                    continue
+
+                endpoint_class = rest_endpoint_config.endpoints_class or ResourceRestEndpoints
+                module.endpoints.append(endpoint_class(resource_config, rest_endpoint_config))
+
+            for endpoint in module.endpoints or []:
+                self.wsgi.register_endpoint(endpoint)
+
         # then init all modules
         for module in self.get_module_list():
             if module.init is not None: