[SDESK-7326] Migrate from Flask to Quart (#2661)

* Upgrade superdesk module

* Upgrade apps module

* Upgrade content_api module

* Update superdesk-core module config files

* Upgrade tests to async

* use async app_context in celery run task

* fix tests

* disable celery tests for now

* add egg name to dev-requirements for planning

* Return Endpoint from EndpointGroup.endpoint wrapper

* use planning/async branch

* Remove debug code and unused import

apps/archive/__init__.py

@@ -17,6 +17,7 @@
 import superdesk
 from superdesk.celery_app import celery
 
+from .common import ARCHIVE  # noqa  - fixes circular imports
 from .archive import (
     ArchiveResource,
     ArchiveService,
@@ -39,7 +40,7 @@
 from apps.common.models.io.eve_proxy import EveProxy
 from .archive_rewrite import ArchiveRewriteResource, ArchiveRewriteService
 from .news import NewsResource, NewsService
-from flask_babel import _, lazy_gettext
+from quart_babel import lazy_gettext
 
 logger = logging.getLogger(__name__)
 

apps/archive/archive.py

@@ -96,7 +96,7 @@
 from .usage import track_usage, update_refs
 from superdesk.utc import utcnow
 from superdesk.vocabularies import is_related_content
-from flask_babel import _
+from quart_babel import gettext as _
 from apps.archive.highlights_search_mixin import HighlightsSearchMixin
 
 EDITOR_KEY_PREFIX = "editor_"
@@ -867,6 +867,7 @@ def update(self, id, updates, original):
             self.deschedule_item(updates, original)  # this is an deschedule action
 
         # send signal
+        # TODO-ASYNC: Support async signals
         signals.item_update.send(self, updates=updates, original=original)
 
         super().update(id, updates, original)
@@ -876,6 +877,7 @@ def update(self, id, updates, original):
         signals.item_updated.send(self, item=updated, original=original)
 
         if "marked_for_user" in updates:
+            # TODO-ASYNC: Support async (see superdesk.tests.markers.requires_eve_resource_async_event)
             self.handle_mark_user_notifications(updates, original)
 
     def deschedule_item(self, updates, original):
@@ -1210,7 +1212,7 @@ def get_expired_items(self, expiry_datetime, last_id=None, invalid_only=False):
         else:
             logger.warning("get_expired_items did not finish in %d loops", get_app_config("MAX_EXPIRY_LOOPS"))
 
-    def handle_mark_user_notifications(self, updates, original, add_activity=True):
+    async def handle_mark_user_notifications(self, updates, original, add_activity=True):
         """Notify user when item is marked or unmarked
 
         :param updates: updates to item that should be saved
@@ -1235,7 +1237,7 @@ def handle_mark_user_notifications(self, updates, original, add_activity=True):
                 headline=original.get("headline", original.get("slugline", "item")), by_user=by_user
             )
 
-            self._send_mark_user_notifications(
+            await self._send_mark_user_notifications(
                 "item:marked",
                 message,
                 resource=self.datasource,
@@ -1255,7 +1257,7 @@ def handle_mark_user_notifications(self, updates, original, add_activity=True):
                 by_user=by_user,
             )
 
-            self._send_mark_user_notifications(
+            await self._send_mark_user_notifications(
                 "item:marked",
                 message,
                 resource=self.datasource,
@@ -1265,7 +1267,7 @@ def handle_mark_user_notifications(self, updates, original, add_activity=True):
                 marked_for_user=marked_for_user,
             )
 
-    def _send_mark_user_notifications(
+    async def _send_mark_user_notifications(
         self, activity_name, msg, resource=None, item=None, user_list=None, add_activity=True, **data
     ):
         """Send notifications on mark or unmark user operation
@@ -1291,7 +1293,7 @@ def _send_mark_user_notifications(
         link = "{}/#/workspace?item={}&action=view".format(client_url, link_id)
 
         if add_activity:
-            notify_and_add_activity(
+            await notify_and_add_activity(
                 activity_name,
                 msg,
                 resource=resource,

apps/archive/archive_correction.py

@@ -22,7 +22,7 @@
 from superdesk.workflow import is_workflow_state_transition_valid
 from superdesk.errors import SuperdeskApiError, InvalidStateTransitionError
 from superdesk.notification import push_notification
-from flask_babel import _
+from quart_babel import gettext as _
 
 logger = logging.getLogger(__name__)
 

apps/archive/archive_link.py

@@ -22,7 +22,7 @@
 from superdesk.errors import SuperdeskApiError
 from superdesk.notification import push_notification
 import logging
-from flask_babel import _
+from quart_babel import gettext as _
 
 logger = logging.getLogger(__name__)
 

apps/archive/archive_rewrite.py

@@ -48,7 +48,7 @@
 from superdesk.signals import item_rewrite
 from apps.archive.archive import update_associations
 from superdesk.editor_utils import generate_fields, copy_fields
-from flask_babel import _
+from quart_babel import gettext as _
 from superdesk.utc import utcnow
 
 logger = logging.getLogger(__name__)

apps/archive/archive_spike.py

@@ -36,7 +36,7 @@
 from superdesk.metadata.packages import LINKED_IN_PACKAGES, PACKAGE
 from superdesk.utc import get_expiry_date, utcnow
 from apps.item_lock.components.item_lock import push_unlock_notification
-from flask_babel import _
+from quart_babel import gettext as _
 from bson.objectid import ObjectId
 
 logger = logging.getLogger(__name__)

apps/archive/autocomplete.py

@@ -2,7 +2,7 @@
 import warnings
 import superdesk
 
-from flask_babel import _
+from quart_babel import gettext as _
 from datetime import timedelta
 
 from superdesk.core import get_current_app, get_app_config

apps/archive/common.py

@@ -49,7 +49,7 @@
 from superdesk.errors import SuperdeskApiError, IdentifierGenerationError
 from superdesk.logging import logger
 from apps.auth import get_user, get_auth  # noqa
-from flask_babel import _
+from quart_babel import gettext as _
 
 
 logger = logging.getLogger(__name__)

apps/archive_broadcast/__init__.py

@@ -10,7 +10,7 @@
 
 import logging
 
-from flask_babel import lazy_gettext
+from quart_babel import lazy_gettext
 import superdesk
 from .broadcast import ArchiveBroadcastResource, ArchiveBroadcastService, ARCHIVE_BROADCAST_NAME
 

apps/archive_broadcast/broadcast.py

@@ -28,7 +28,7 @@
 from apps.archive.archive import SOURCE
 from apps.publish.content.common import ITEM_CORRECT, ITEM_PUBLISH
 from superdesk.utc import utcnow
-from flask_babel import _
+from quart_babel import gettext as _
 
 
 logger = logging.getLogger(__name__)

apps/archived/archived.py

@@ -52,7 +52,7 @@
 from superdesk.resource import Resource
 from superdesk.utc import utcnow
 from apps.publish.content import KillPublishService, TakeDownPublishService
-from flask_babel import _, lazy_gettext
+from quart_babel import gettext as _, lazy_gettext
 
 logger = logging.getLogger(__name__)
 PACKAGE_TYPE = "package_type"
@@ -236,6 +236,7 @@ def update(self, id, updates, original):
 
         for article in articles_to_kill:
             updates_copy = deepcopy(updates)
+            # TODO-ASYNC: Support async (see superdesk.tests.markers.requires_eve_resource_async_event)
             kill_service.apply_kill_override(article, updates_copy)
             updated.update(updates_copy)
             # Step 2, If it is flagged as archived only it has no related items in the system so can be deleted.

apps/auth/__init__.py

@@ -9,7 +9,7 @@
 # at https://www.sourcefabric.org/superdesk/license
 
 import logging
-from flask_babel import _
+from quart_babel import gettext as _
 
 from superdesk.resource_fields import ID_FIELD
 from superdesk.flask import g

apps/auth/auth.py

@@ -26,7 +26,7 @@
     get_intrinsic_privileges,
 )
 from superdesk.utc import utcnow
-from flask_babel import _
+from quart_babel import gettext as _
 
 logger = logging.getLogger(__name__)
 
@@ -147,6 +147,7 @@ def check_auth(self, token, allowed_roles, resource, method):
         auth_service = get_resource_service("auth")
         user_service = get_resource_service("users")
         auth_token = auth_service.find_one(token=token, req=None)
+        # TODO-ASYNC: Fix auth token passing in headers (see superdesk.tests.markers.requires_auth_headers_fix)
         if auth_token:
             if session.get("session_token") != token:
                 session["session_token"] = token

apps/auth/db/commands.py

@@ -42,7 +42,7 @@ class CreateUserCommand(superdesk.Command):
         superdesk.Option("--support", "-s", dest="support", required=False, action="store_true"),
     ]
 
-    def run(self, username, password, email, admin=False, support=False):
+    async def run(self, username, password, email, admin=False, support=False):
         # force type conversion to boolean
         user_type = "administrator" if admin else "user"
 
@@ -57,7 +57,7 @@ def run(self, username, password, email, admin=False, support=False):
         }
 
         app = get_current_app().as_any()
-        with app.test_request_context("/users", method="POST"):
+        async with app.test_request_context("/users", method="POST"):
             if userdata.get("password", None) and not is_hashed(userdata.get("password")):
                 userdata["password"] = get_hash(
                     userdata.get("password"), get_app_config("BCRYPT_GENSALT_WORK_FACTOR", 12)
@@ -158,12 +158,16 @@ def run(self, fields, import_file, activation_email=False):
         created_users = 0
 
         for user_data in data:
+            print("Step 1")
+            print("Processing user")
+            print(user_data)
             if not isinstance(user_data, dict):
                 self.parser.error(
                     "Invalid user data when importing {path!r}: user data must be an object, not {data_type}:\ndata: "
                     "{data!r}".format(path=import_file, data_type=type(user_data), data=user_data)
                 )
 
+            print("Step 2")
             try:
                 username = user_data["username"]
                 email = user_data["email"]
@@ -176,6 +180,7 @@ def run(self, fields, import_file, activation_email=False):
                 )
                 continue
 
+            print("Step 3")
             diff_fields = set(user_data) - USER_FIELDS_NAMES
             if diff_fields:
                 logger.warning(
@@ -185,8 +190,10 @@ def run(self, fields, import_file, activation_email=False):
                     )
                 )
 
+            print("Step 4")
             clean_data = {"needs_activation": activation_email}
             try:
+                print("Step 4.1")
                 for field_name in USER_FIELDS_NAMES.intersection(user_data):
                     value = user_data[field_name]
                     if field_name == "role":
@@ -204,12 +211,13 @@ def run(self, fields, import_file, activation_email=False):
 
                 user_id = users_service.post([clean_data])[0]
             except Exception as e:
-                logger.warning(
+                logger.exception(
                     "Can't create user {username!r}: {reason}\n{data!r}".format(username=username, reason=e, data=data)
                 )
                 continue
             logger.info("user {username!r} created with id '{user_id}'".format(username=username, user_id=user_id))
 
+            print("Step 5")
             created_users += 1
 
         print(

apps/auth/db/db.py

@@ -16,7 +16,7 @@
 from apps.auth.errors import CredentialsAuthError, PasswordExpiredError, ExternalUserError
 from superdesk.utc import utcnow
 import datetime
-from flask_babel import _
+from quart_babel import gettext as _
 
 
 class DbAuthService(AuthService):

apps/auth/db/reset_password.py

@@ -20,7 +20,7 @@
 from superdesk.utils import get_random_string
 from superdesk.emails import send_reset_password_email
 from superdesk.errors import SuperdeskApiError
-from flask_babel import _
+from quart_babel import gettext as _
 
 logger = logging.getLogger(__name__)
 
@@ -71,6 +71,7 @@ def create(self, docs, **kwargs):
                 return self.reset_password(doc)
             if email:
                 email = email.lower()
+                # TODO-ASYNC: Support async (see superdesk.tests.markers.requires_eve_resource_async_event)
                 return self.initialize_reset_password(doc, email)
             if key:
                 token_req = self.check_if_valid_token(key)
@@ -88,7 +89,7 @@ def store_reset_password_token(self, doc, email, days_alive, user_id):
         ids = super().create([doc])
         return ids
 
-    def initialize_reset_password(self, doc, email):
+    async def initialize_reset_password(self, doc, email):
         token_ttl = get_app_config("RESET_PASSWORD_TOKEN_TIME_TO_LIVE")
 
         user = superdesk.get_resource_service("users").find_one(req=None, email=email)
@@ -105,7 +106,7 @@ def initialize_reset_password(self, doc, email):
             raise SuperdeskApiError.forbiddenError(_("User not active"))
 
         ids = self.store_reset_password_token(doc, email, token_ttl, user["_id"])
-        send_reset_password_email(doc, token_ttl)
+        await send_reset_password_email(doc, token_ttl)
         self.remove_private_data(doc)
         return ids
 

apps/auth/oidc/auth.py

@@ -8,7 +8,7 @@
 # AUTHORS and LICENSE files distributed with this source code, or
 # at https://www.sourcefabric.org/superdesk/license
 
-from flask_babel import _
+from quart_babel import gettext as _
 
 # from flask_oidc_ex import OpenIDConnect
 

apps/auth/service.py

@@ -8,7 +8,7 @@
 # AUTHORS and LICENSE files distributed with this source code, or
 # at https://www.sourcefabric.org/superdesk/license
 
-from flask_babel import _
+from quart_babel import gettext as _
 
 from superdesk.core import get_current_app
 from superdesk.resource_fields import ID_FIELD

apps/comments/__init__.py

@@ -25,4 +25,5 @@ def init_app(app) -> None:
     app.on_updated_activity -= on_activity_updated
     app.on_updated_activity += on_activity_updated
 
+    # TODO-ASYNC: Support async events
     superdesk.item_update.connect(handle_inline_mentions)

apps/comments/comments.py

@@ -14,7 +14,7 @@
 from superdesk.services import BaseService
 from superdesk.errors import SuperdeskApiError
 from .user_mentions import get_users, get_desks, get_mentions, notify_mentioned_users, notify_mentioned_desks
-from flask_babel import _
+from quart_babel import gettext as _
 
 comments_schema = {
     "text": {
@@ -101,6 +101,7 @@ def on_created(self, docs):
             decode_keys(doc, "mentioned_desks")
 
         if self.notifications:
+            # TODO-ASYNC: Support async (see superdesk.tests.markers.requires_eve_resource_async_event)
             notify_mentioned_users(docs, get_app_config("CLIENT_URL", "").rstrip("/"))
             notify_mentioned_desks(docs)
 

apps/comments/inline_comments.py

@@ -10,7 +10,7 @@
 USER_MENTIONS_REGEX = re.compile(r"@\[([^]]+)\]\(user:([a-f0-9]{24})\)")
 
 
-def handle_inline_mentions(sender, updates, original):
+async def handle_inline_mentions(sender, updates, original):
     """Listen to item_update signal and send notifications to new inline user mentions."""
     updated = original.copy()
     updated.update(updates)
@@ -19,7 +19,7 @@ def handle_inline_mentions(sender, updates, original):
         if not comment.get("notified"):
             users = _get_mentioned_users(comment)
             if users:
-                notify_mentioned_users(
+                await notify_mentioned_users(
                     [
                         {
                             "_id": "",