feature/3991_weekly_email_alert

doajtest/testbook/article_deletion_notifications/article_deletion_notifications.yml

@@ -0,0 +1,27 @@
+suite: Article Deletion Notifications
+testset: Article Deletion Notifications
+
+tests:
+  - title: Article Deletion Notification Generation
+    setup:
+      - As an admin or through the API, delete one or more articles belonging to a publisher.
+      - Ensure the articles were deleted within the last 7 days.
+    context:
+      role: admin
+    steps:
+      - step: Log in as the publisher who owned the deleted articles after an hour (scheduler configured to run hourly for testing).
+      - step: Check the notifications (bell icon or /notifications).
+        results:
+          - A new notification is present with the title "Deleted articles in your journal(s) this week".
+          - The notification content lists the titles and details of the deleted articles.
+      - step: Verify that if multiple articles were deleted for the same publisher, they are grouped into a single notification.
+
+  - title: No Notifications when no articles deleted
+    setup:
+      - Ensure no articles have been deleted in the last 7 days (or for a specific set of publishers).
+    context:
+      role: admin
+    steps:
+      - step: Check notifications for publishers.
+        results:
+          - No new deletion notifications are received.

doajtest/testbook/journal_form/maned_form.yml

@@ -180,7 +180,7 @@ tests:
     role: admin
   steps:
     - step: Go to admin journal search page at /admin/journals
-    - step: "Locate a record with the metadata value 'Last full review: Never'" and click "Edit this journal"
+    - step: "Locate a record with the metadata value 'Last full review: Never' and click 'Edit this journal'"
     - step: Locate the "Last Full Review" section of the form
     - step: Enter a date in the future in "Last Full Review" field
       results:
@@ -202,7 +202,7 @@ tests:
     role: admin
   steps:
     - step: Go to admin journal search page at /admin/journals
-    - step: "Locate a record with the metadata value 'Last Owner Transfer: Never'" and click "Edit this journal"
+    - step: "Locate a record with the metadata value 'Last Owner Transfer: Never' and click 'Edit this journal'"
     - step: Locate the "Re-assign publisher account" section of the form
     - step: Change the user to a different user account
     - step: Save the journal record

doajtest/unit/test_tasks_article_deletion_notifications.py

@@ -0,0 +1,91 @@
+from doajtest.helpers import DoajTestCase
+from portality.tasks.article_deletion_notifications import ArticleDeletionNotificationsBackgroundTask
+from portality import models
+from doajtest.fixtures.article import ArticleFixtureFactory
+from doajtest.fixtures.accounts import AccountFixtureFactory
+from portality.lib import dates
+import time
+
+class TestArticleDeletionNotifications(DoajTestCase):
+
+    def test_article_deletion_notifications(self):
+        # 1. Setup data
+        # Create two publishers
+        pub1 = models.Account(**AccountFixtureFactory.make_publisher_source())
+        pub1.set_id("pub1")
+        pub1.save(blocking=True)
+
+        pub2 = models.Account(**AccountFixtureFactory.make_publisher_source())
+        pub2.set_id("pub2")
+        pub2.save(blocking=True)
+
+        # Create some ArticleTombstones
+        # Recent tombstone for pub1
+        at1 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source())
+        at1.set_id("at1")
+        at1.set_created(dates.format(dates.before_now(1 * 24 * 60 * 60)))
+        at1.data['admin'] = {'owner': 'pub1'}
+        at1.save(blocking=True)
+
+        # Another recent tombstone for pub1
+        at2 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source())
+        at2.set_id("at2")
+        at2.set_created(dates.format(dates.before_now(2 * 24 * 60 * 60)))
+        at2.data['admin'] = {'owner': 'pub1'}
+        at2.save(blocking=True)
+
+        # Recent tombstone for pub2
+        at3 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source())
+        at3.set_id("at3")
+        at3.set_created(dates.format(dates.before_now(3 * 24 * 60 * 60)))
+        at3.data['admin'] = {'owner': 'pub2'}
+        at3.save(blocking=True)
+
+        # Old tombstone (should be ignored)
+        at4 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source())
+        at4.set_id("at4")
+        at4.set_created(dates.format(dates.before_now(10 * 24 * 60 * 60)))
+        at4.data['admin'] = {'owner': 'pub1'}
+        at4.save(blocking=True)
+
+        # 2. Run the task
+        job = ArticleDeletionNotificationsBackgroundTask.prepare("system")
+        task = ArticleDeletionNotificationsBackgroundTask(job)
+        task.run()
+
+        time.sleep(2)
+
+        # 3. Verify notifications
+        # Pub1 should have 1 notification with 2 articles
+        notes1 = models.Notification.all()
+        self.assertEqual(len(notes1), 2)
+
+        note1 = None
+        for n in notes1:
+            if n.who == "pub1":
+                note1 = n
+                break
+
+        self.assertIsNotNone(note1)
+        self.assertIn("Deleted articles in your journal(s) this week", note1.short)
+        # It should contain titles of at1 and at2, but not at4
+        # We need to be careful about what make_article_source produces as titles
+
+        # Pub2 should have 1 notification with 1 article
+        note2 = None
+        for n in notes1:
+            if n.who == "pub2":
+                note2 = n
+                break
+        self.assertIsNotNone(note2)
+
+    def test_no_articles(self):
+        # Run task without any tombstones
+        job = ArticleDeletionNotificationsBackgroundTask.prepare("system")
+        task = ArticleDeletionNotificationsBackgroundTask(job)
+        task.run()
+
+        # Verify no notifications sent
+        notes = models.Notification.all()
+        self.assertEqual(len(notes), 0)
+        self.assertIn("No deleted articles in the last week", job.audit[0]['message'])

portality/models/article.py

@@ -223,11 +223,32 @@ def snapshot(self):
     def _tombstone(self):
         stone = ArticleTombstone()
         stone.set_id(self.id)
-        sbj = stone.bibjson()
 
+        # Copy full bibjson so we retain title, authors, journal title, volume/number and pages
+        try:
+            sbj_src = deepcopy(self.data.get("bibjson", {}))
+        except Exception:
+            sbj_src = self.data.get("bibjson", {})
+        stone.set_bibjson(sbj_src)
+
+        # Ensure subjects are present (for back-compat with any consumers expecting it)
         subs = self.bibjson().subjects()
-        for s in subs:
-            sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code"))
+        if subs is not None:
+            sbj = stone.bibjson()
+            for s in subs:
+                sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code"))
+
+        # Add admin metadata, in particular owner
+        admin = deepcopy(self.data.get("admin", {})) if isinstance(self.data.get("admin"), dict) else {}
+        if "owner" not in admin or admin.get("owner") is None:
+            # Attempt to resolve owner via associated journals
+            try:
+                admin["owner"] = self.get_owner()
+            except Exception:
+                # If we can't determine owner, leave as-is
+                pass
+        if len(admin) > 0:
+            stone.data["admin"] = admin
 
         stone.save()
         return stone

portality/settings.py

@@ -451,6 +451,8 @@
     "datalog_journal_added_update": {"month": "*", "day": "*", "day_of_week": "*", "hour": "4", "minute": "30"},
     "auto_assign_editor_group_data": {"month": "*", "day": "*/7", "day_of_week": "*", "hour": "3", "minute": "30"},
     "ris_export": {"month": "*", "day": "15", "day_of_week": "*", "hour": "3", "minute": "30"},
+    # Weekly notification to publishers about deleted articles (Article Tombstones)
+    "article_deletion_notifications": {"month": "*", "day": "*", "day_of_week": "1", "hour": "5", "minute": "10"},
 }
 
 

portality/tasks/article_deletion_notifications.py

@@ -0,0 +1,157 @@
+"""Weekly notifications to publishers about articles deleted in the last week.
+Collect article details from ArticleTombstone and send a Notification per owner.
+"""
+from typing import Dict, List
+
+from portality import models, constants
+from portality.background import BackgroundTask, BackgroundApi
+from portality.bll.doaj import DOAJ
+from portality.core import app
+from portality.lib import dates
+from portality.tasks.helpers import background_helper
+from portality.tasks.redis_huey import scheduled_short_queue as queue
+
+
+class ArticleDeletionNotificationsBackgroundTask(BackgroundTask):
+    __action__ = "article_deletion_notifications"
+
+    def run(self):
+        job = self.background_job
+
+        # Determine date range: last 7 days
+        since = dates.before_now(7 * 24 * 60 * 60)
+        since_str = dates.format(since)
+
+        # Query tombstones created in the last week
+        q = {
+            "query": {
+                "range": {
+                    "created_date": {"gte": since_str}
+                }
+            }
+        }
+
+        owner_to_items: Dict[str, List[dict]] = {}
+
+        for stone in models.ArticleTombstone.iterate(q, page_size=1000):
+            admin = getattr(stone, 'data', {}).get('admin', {}) if hasattr(stone, 'data') else {}
+            owner = admin.get('owner')
+            if not owner:
+                continue
+
+            bj = stone.bibjson()
+            title = bj.title if hasattr(bj, 'title') else stone.data.get('bibjson', {}).get('title')
+            authors = []
+            try:
+                for a in bj.author or []:
+                    n = a.get('name') if isinstance(a, dict) else None
+                    if n:
+                        authors.append(n)
+            except Exception:
+                pass
+
+            volume = bj.volume if hasattr(bj, 'volume') else None
+            issue = bj.number if hasattr(bj, 'number') else None
+            start_page = bj.start_page if hasattr(bj, 'start_page') else None
+            end_page = bj.end_page if hasattr(bj, 'end_page') else None
+            pages = None
+            if start_page and end_page:
+                pages = f"{start_page}-{end_page}"
+            elif start_page:
+                pages = str(start_page)
+
+            issns = []
+            try:
+                issns = bj.issns()
+            except Exception:
+                pass
+
+            journal_name = None
+            try:
+                journal_name = bj.journal_title
+            except Exception:
+                # fall back to raw
+                journal_name = stone.data.get('bibjson', {}).get('journal', {}).get('title')
+
+            item = {
+                'title': title,
+                'authors': authors,
+                'volume': volume,
+                'issue': issue,
+                'pages': pages,
+                'issns': issns,
+                'journal': journal_name,
+            }
+
+            owner_to_items.setdefault(owner, []).append(item)
+
+        if not owner_to_items:
+            job.add_audit_message("No deleted articles in the last week; no notifications sent.")
+            return
+
+        notify_svc = DOAJ.notificationsService()
+
+        total_notes = 0
+        for owner, items in owner_to_items.items():
+            acc = models.Account.pull(owner)
+            if acc is None:
+                job.add_audit_message(f"Owner account {owner} not found; skipping {len(items)} items")
+                continue
+
+            # Build notification content
+            short = "Deleted articles in your journal(s) this week"
+
+            lines = [
+                "The following articles have been deleted from DOAJ in the last week:",
+                "",
+            ]
+            for it in items:
+                auth = (", ".join(it['authors'])) if it['authors'] else "Unknown author"
+                issn_str = ", ".join([i for i in (it.get('issns') or []) if i])
+                vol_str = f", vol {it['volume']}" if it.get('volume') else ""
+                iss_str = f", issue {it['issue']}" if it.get('issue') else ""
+                pg_str = f", pages {it['pages']}" if it.get('pages') else ""
+                jn = f" ({it['journal']})" if it.get('journal') else ""
+                line = f"- {it['title'] or 'Untitled'} by {auth}{vol_str}{iss_str}{pg_str} [{issn_str}]{jn}"
+                lines.append(line)
+            lines.append("")
+            lines.append("Please upload replacement records if appropriate.")
+
+            note = models.Notification()
+            note.who = owner
+            note.classification = constants.NOTIFICATION_CLASSIFICATION_STATUS
+            note.short = short
+            note.long = "\n".join(lines)
+
+            notify_svc.notify(note)
+            total_notes += 1
+
+        job.add_audit_message(f"Sent {total_notes} publisher deletion notifications.")
+
+    def cleanup(self):
+        pass
+
+    @classmethod
+    def prepare(cls, username, **kwargs):
+        job = background_helper.create_job(username, cls.__action__,
+                                           queue_id=huey_helper.queue_id)
+        return job
+
+    @classmethod
+    def submit(cls, background_job):
+        background_job.save()
+        execute_article_deletion_notifications.schedule(args=(background_job.id,),
+                                                        delay=app.config.get('HUEY_ASYNC_DELAY', 10))
+
+
+huey_helper = ArticleDeletionNotificationsBackgroundTask.create_huey_helper(queue)
+
+
+@huey_helper.register_schedule
+def scheduled_article_deletion_notifications():
+    background_helper.submit_by_bg_task_type(ArticleDeletionNotificationsBackgroundTask)
+
+
+@huey_helper.register_execute(is_load_config=False)
+def execute_article_deletion_notifications(job_id):
+    background_helper.execute_by_job_id(job_id, lambda j: ArticleDeletionNotificationsBackgroundTask(j))

portality/tasks/consumer_scheduled_short_queue.py

@@ -23,3 +23,4 @@
 from portality.tasks.reporting import scheduled_reports, run_reports  # noqa
 from portality.tasks.request_es_backup import scheduled_request_es_backup, request_es_backup  # noqa
 from portality.tasks.auto_assign_editor_group_data import scheduled_auto_assign_editor_group_data, auto_assign_editor_group_data
+from portality.tasks.article_deletion_notifications import scheduled_article_deletion_notifications, execute_article_deletion_notifications

test.cfg

@@ -59,6 +59,7 @@ HUEY_SCHEDULE = {
     "datalog_journal_added_update": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "*/30"},
     "auto_assign_editor_group_data": {"month": "*", "day": "*/7", "day_of_week": "*", "hour": "3", "minute": "30"},
     "ris_export": CRON_NEVER,
+    "article_deletion_notifications": {"month": "*", "day": "*", "day_of_week": "1", "hour": "5", "minute": "10"},
 }
 
 # =======================