diff --git a/cms/data/notifications.yml b/cms/data/notifications.yml index e918780d80..e9d52d57b0 100644 --- a/cms/data/notifications.yml +++ b/cms/data/notifications.yml @@ -128,6 +128,16 @@ bg:job:admin_reports:export_available: short: Report "{name}" ready for download +bg:job:article_deletion_notifications:publisher: + long: | + The following articles have been deleted from DOAJ in the last week + + {list_of_articles} + + Please upload replacement records if appropriate. + short: + Deleted articles in your journal(s) this week + journal:assed:assigned:notify: long: | The journal **{journal_name}** has been assigned to you by the Editor of your group **{group_name}**. Please start work on this within 10 days. diff --git a/dev.template.cfg b/dev.template.cfg index dd70ea82bb..b0b1677eaa 100644 --- a/dev.template.cfg +++ b/dev.template.cfg @@ -87,7 +87,8 @@ HUEY_SCHEDULE = { "datalog_journal_added_update": CRON_NEVER, "auto_assign_editor_group_data": CRON_NEVER, "ris_export": CRON_NEVER, - "site_statistics": CRON_NEVER + "site_statistics": CRON_NEVER, + "article_deletion_notifications": CRON_NEVER, } ########################################### diff --git a/doajtest/testbook/article_deletion_notifications/article_deletion_notifications.yml b/doajtest/testbook/article_deletion_notifications/article_deletion_notifications.yml new file mode 100644 index 0000000000..09c3dbb791 --- /dev/null +++ b/doajtest/testbook/article_deletion_notifications/article_deletion_notifications.yml @@ -0,0 +1,24 @@ +suite: Article Deletion Notifications +testset: Article Deletion Notifications + +tests: + - title: Article Deletion Notification Generation + context: + role: admin + steps: + - step: Run the testdrive for deleted_articles to set up the fixtures for this test + path: /testdrive/deleted_articles + - step: Log in to the system as an administrator + - step: Go to the admin notifications page + path: /admin/notifications + results: + - There are no notifications listed for either user account supplied in the testdrive + - step: Go to the testdrive page without reloading it, and click the button marked "Run background task". + results: + - The text beneath the button confirms that the script executed successfully + - step: Return to the admin notifications page + results: + - A new notification is present with the title "Deleted articles in your journal(s) this week". + - The notification content lists the titles and details of the deleted articles. + - There is only one notification for the first publisher listed on the testdrive. The second publisher + does not have a notification diff --git a/doajtest/testbook/journal_form/maned_form.yml b/doajtest/testbook/journal_form/maned_form.yml index 22a88581d3..a5c442b4be 100644 --- a/doajtest/testbook/journal_form/maned_form.yml +++ b/doajtest/testbook/journal_form/maned_form.yml @@ -180,7 +180,7 @@ tests: role: admin steps: - step: Go to admin journal search page at /admin/journals - - step: "Locate a record with the metadata value 'Last full review: Never'" and click "Edit this journal" + - step: "Locate a record with the metadata value 'Last full review: Never' and click 'Edit this journal'" - step: Locate the "Last Full Review" section of the form - step: Enter a date in the future in "Last Full Review" field results: @@ -202,7 +202,7 @@ tests: role: admin steps: - step: Go to admin journal search page at /admin/journals - - step: "Locate a record with the metadata value 'Last Owner Transfer: Never'" and click "Edit this journal" + - step: "Locate a record with the metadata value 'Last Owner Transfer: Never' and click 'Edit this journal'" - step: Locate the "Re-assign publisher account" section of the form - step: Change the user to a different user account - step: Save the journal record diff --git a/doajtest/testdrive/deleted_articles.py b/doajtest/testdrive/deleted_articles.py new file mode 100644 index 0000000000..31741c07bf --- /dev/null +++ b/doajtest/testdrive/deleted_articles.py @@ -0,0 +1,36 @@ +from doajtest.testdrive.factory import TestDrive + + +class DeletedArticles(TestDrive): + def setup(self) -> dict: + # create 2 publisher accounts. The first will have tombstoned articles, + # the second will not + publisher1, pw1 = self.publisher_account() + publisher2, pw2 = self.publisher_account() + + # create one journal for each publisher + journals1 = self.journals_in_doaj(publisher1, n=1, block=True) + journals2 = self.journals_in_doaj(publisher2, n=1, block=True) + + # place 10 articles in each journal + articles1 = self.articles(journals1[0], n=10) + articles2 = self.articles(journals2[0], n=10) + + # for the first publishers journal, tombstone 10 articles + tombs1 = self.article_tombstones(journals1[0], n=10) + + report = {} + self.report_accounts([(publisher1, pw1), (publisher2, pw2)], report) + self.report_journal_ids(journals1 + journals2, report) + self.report_article_ids(articles1 + articles2, report) + self.report_article_tombstone_ids(tombs1, report) + self.report_script("article_deletion_notifications", "Run background task", report) + + return report + + def teardown(self, params) -> dict: + self.teardown_accounts(params) + self.teardown_journals(params) + self.teardown_articles(params) + self.teardown_article_tombstones(params) + return self.SUCCESS \ No newline at end of file diff --git a/doajtest/testdrive/factory.py b/doajtest/testdrive/factory.py index 28149151cf..690d555f9d 100644 --- a/doajtest/testdrive/factory.py +++ b/doajtest/testdrive/factory.py @@ -1,10 +1,18 @@ from portality.lib import plugin import random import string -from portality import models +from portality import models, constants +from doajtest.fixtures.v2.journals import JournalFixtureFactory +from doajtest.fixtures.article import ArticleFixtureFactory class TestDrive(): + + SUCCESS = {"status": "success"} + + def __init__(self): + self.run_seed = self.create_random_str() + def create_random_str(self, n_char=10): s = string.ascii_letters + string.digits return ''.join(random.choices(s, k=n_char)) @@ -22,6 +30,115 @@ def setup(self) -> dict: def teardown(self, setup_params) -> dict: return {"status": "not implemented"} + ### Reporting functions for consistent delivery of similar data models + + def report_accounts(self, accounts, report): + report["accounts"] = [] + for acc, pw in accounts: + report["accounts"].append({ + "username": acc.id, + "password": pw, + "api_key": acc.api_key + }) + + def report_journal_ids(self, journals, report): + report["journals"] = [] + for j in journals: + report["journals"].append(j.id) + + def report_article_ids(self, articles, report): + report["articles"] = [] + for a in articles: + report["articles"].append(a.id) + + def report_article_tombstone_ids(self, tombstones, report): + report["article_tombstones"] = [] + for a in tombstones: + report["article_tombstones"].append(a.id) + + def report_script(self, script, name, report): + report["script"] = { + "script_name": script, + "title": name + } + + ### Teardown methods for consistent cleanup of similar data models + + def teardown_accounts(self, report): + for acc in report.get("accounts", []): + models.Account.remove_by_id(acc["username"]) + + def teardown_journals(self, report): + for jid in report.get("journals", []): + models.Journal.remove_by_id(jid) + + def teardown_articles(self, report): + for aid in report.get("articles", []): + models.Article.remove_by_id(aid) + + def teardown_article_tombstones(self, report): + for tid in report.get("article_tombstones", []): + models.ArticleTombstone.remove_by_id(tid) + + ### Useful factory methods + + def publisher_account(self, save=True, block=False): + un = self.create_random_str() + pw = self.create_random_str() + acc = models.Account.make_account(un + "@example.com", un, "Publisher " + un, [constants.ROLE_PUBLISHER, constants.ROLE_API]) + acc.set_password(pw) + acc.generate_api_key() + if save: + acc.save(blocking=block) + return acc, pw + + def journals_in_doaj(self, owner, n=1, save=True, block=False): + journals = [] + for i in range(n): + template = { + "bibjson": { + "title": f"Journal {owner} {i} {self.run_seed}", + } + } + source = JournalFixtureFactory.make_journal_source(in_doaj=True, overlay=template) + j = models.Journal(**source) + j.remove_current_application() + j.set_id(j.makeid()) + j.set_owner(owner.id) + j.bibjson().eissn = self.generate_unique_issn() + j.bibjson().pissn = self.generate_unique_issn() + if save: + j.save(blocking=block) + journals.append(j) + + return journals + + def articles(self, journal, n=1, save=True): + articles = [] + for i in range(n): + eissn = journal.bibjson().eissn + pissn = journal.bibjson().pissn + a = ArticleFixtureFactory.make_article_source(eissn=eissn, pissn=pissn, with_id=True, in_doaj=journal.is_in_doaj()) + a = models.Article(**a) + a.bibjson().title = f"Article {journal.owner} {i} {self.run_seed}" + a.set_id(a.makeid()) + if save: + a.save() + articles.append(a) + return articles + + def article_tombstones(self, journal, n=1, save=True): + articles = self.articles(journal, n=n, save=False) + + tombs = [] + for a in articles: + t = a.make_tombstone() + if save: + t.save() + tombs.append(t) + + return tombs + class TestFactory(): @classmethod diff --git a/doajtest/unit/test_tasks_article_deletion_notifications.py b/doajtest/unit/test_tasks_article_deletion_notifications.py new file mode 100644 index 0000000000..54b3f13319 --- /dev/null +++ b/doajtest/unit/test_tasks_article_deletion_notifications.py @@ -0,0 +1,91 @@ +from doajtest.helpers import DoajTestCase +from portality.tasks.article_deletion_notifications import ArticleDeletionNotificationsBackgroundTask +from portality import models +from doajtest.fixtures.article import ArticleFixtureFactory +from doajtest.fixtures.accounts import AccountFixtureFactory +from portality.lib import dates +import time + +class TestArticleDeletionNotifications(DoajTestCase): + + def test_article_deletion_notifications(self): + # 1. Setup data + # Create two publishers + pub1 = models.Account(**AccountFixtureFactory.make_publisher_source()) + pub1.set_id("pub1") + pub1.save(blocking=True) + + pub2 = models.Account(**AccountFixtureFactory.make_publisher_source()) + pub2.set_id("pub2") + pub2.save(blocking=True) + + # Create some ArticleTombstones + # Recent tombstone for pub1 + at1 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source()) + at1.set_id("at1") + at1.set_created(dates.format(dates.before_now(1 * 24 * 60 * 60))) + at1.data['admin'] = {'owner': 'pub1'} + at1.save(blocking=True) + + # Another recent tombstone for pub1 + at2 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source()) + at2.set_id("at2") + at2.set_created(dates.format(dates.before_now(2 * 24 * 60 * 60))) + at2.data['admin'] = {'owner': 'pub1'} + at2.save(blocking=True) + + # Recent tombstone for pub2 + at3 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source()) + at3.set_id("at3") + at3.set_created(dates.format(dates.before_now(3 * 24 * 60 * 60))) + at3.data['admin'] = {'owner': 'pub2'} + at3.save(blocking=True) + + # Old tombstone (should be ignored) + at4 = models.ArticleTombstone(**ArticleFixtureFactory.make_article_source()) + at4.set_id("at4") + at4.set_created(dates.format(dates.before_now(10 * 24 * 60 * 60))) + at4.data['admin'] = {'owner': 'pub1'} + at4.save(blocking=True) + + # 2. Run the task + job = ArticleDeletionNotificationsBackgroundTask.prepare("system") + task = ArticleDeletionNotificationsBackgroundTask(job) + task.run() + + time.sleep(2) + + # 3. Verify notifications + # Pub1 should have 1 notification with 2 articles + notes1 = models.Notification.all() + self.assertEqual(len(notes1), 2) + + note1 = None + for n in notes1: + if n.who == "pub1": + note1 = n + break + + self.assertIsNotNone(note1) + self.assertIn("Deleted articles in your journal(s) this week", note1.short) + # It should contain titles of at1 and at2, but not at4 + # We need to be careful about what make_article_source produces as titles + + # Pub2 should have 1 notification with 1 article + note2 = None + for n in notes1: + if n.who == "pub2": + note2 = n + break + self.assertIsNotNone(note2) + + def test_no_articles(self): + # Run task without any tombstones + job = ArticleDeletionNotificationsBackgroundTask.prepare("system") + task = ArticleDeletionNotificationsBackgroundTask(job) + task.run() + + # Verify no notifications sent + notes = models.Notification.all() + self.assertEqual(len(notes), 0) + self.assertIn("No deleted articles in the last week", job.audit[0]['message']) diff --git a/portality/models/__init__.py b/portality/models/__init__.py index bdd14687f5..6560b0da33 100644 --- a/portality/models/__init__.py +++ b/portality/models/__init__.py @@ -14,7 +14,7 @@ from portality.models.uploads import FileUpload, ExistsFileQuery, OwnerFileQuery, ValidFileQuery, BulkArticles from portality.models.lock import Lock from portality.models.history import ArticleHistory, JournalHistory -from portality.models.article import Article, ArticleBibJSON, ArticleQuery, ArticleVolumesQuery, DuplicateArticleQuery, NoJournalException, ArticleTombstone +from portality.models.article import Article, ArticleBibJSON, ArticleQuery, ArticleVolumesQuery, DuplicateArticleQuery, NoJournalException, ArticleTombstone, ArticleTombstoneRecentlyDeletedQuery from portality.models.oaipmh import OAIPMHRecord, OAIPMHJournal, OAIPMHArticle from portality.models.atom import AtomRecord from portality.models.search import JournalArticle, JournalStatsQuery, ArticleStatsQuery diff --git a/portality/models/article.py b/portality/models/article.py index 741e9935e5..6c293bfa18 100644 --- a/portality/models/article.py +++ b/portality/models/article.py @@ -220,15 +220,44 @@ def snapshot(self): hist.save() return hist.id - def _tombstone(self): + def make_tombstone(self): stone = ArticleTombstone() stone.set_id(self.id) sbj = stone.bibjson() - subs = self.bibjson().subjects() - for s in subs: - sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code")) + abj = self.bibjson() + + # copy minimal essential metadata + sbj.title = abj.title + sbj.author = abj.author + sbj.volume = abj.volume + sbj.number = abj.number + sbj.start_page = abj.start_page + sbj.end_page = abj.end_page + sbj.journal_title = abj.journal_title + pissn = abj.get_one_identifier(abj.P_ISSN) + if pissn is not None: + sbj.add_identifier(sbj.P_ISSN, pissn) + eissn = abj.get_one_identifier(abj.E_ISSN) + if eissn is not None: + sbj.add_identifier(sbj.E_ISSN, eissn) + + # Ensure subjects are present (for back-compat with any consumers expecting it) + subs = abj.subjects() + if subs is not None: + for s in subs: + sbj.add_subject(s.get("scheme"), s.get("term"), s.get("code")) + + # Add owner (at time of deletion) information + stone.owner = self.get_owner() + + return stone + + def _tombstone(self): + # make the tombstone + stone = self.make_tombstone() + # save and return stone.save() return stone @@ -643,6 +672,16 @@ def snapshot(self): def is_in_doaj(self): return False + @property + def owner(self): + return self.data.get("admin", {}).get("owner") + + @owner.setter + def owner(self, owner): + if "admin" not in self.data: + self.data["admin"] = {} + self.data["admin"]["owner"] = owner + def prep(self): self.data['last_updated'] = dates.now_str() @@ -1174,3 +1213,18 @@ def _sort_articles(articles): s += [x for x in unsorted] return s + + +class ArticleTombstoneRecentlyDeletedQuery(object): + def __init__(self, since=None): + self.since = since + + def query(self): + q = { + "query": { + "range": { + "created_date": {"gte": dates.format(self.since)} + } + } + } + return q \ No newline at end of file diff --git a/portality/scripts/article_deletion_notifications.py b/portality/scripts/article_deletion_notifications.py new file mode 100644 index 0000000000..3d70594cbb --- /dev/null +++ b/portality/scripts/article_deletion_notifications.py @@ -0,0 +1,18 @@ +from portality.core import app +from portality.tasks import article_deletion_notifications +from portality.background import BackgroundApi +from portality.models.background import StdOutBackgroundJob + +def main(): + if app.config.get("SCRIPTS_READ_ONLY_MODE", False): + print("System is in READ-ONLY mode, script cannot run") + exit() + + user = app.config.get("SYSTEM_USERNAME") + job = article_deletion_notifications.ArticleDeletionNotificationsBackgroundTask.prepare(user) + job = StdOutBackgroundJob(job, force_logging=True) + task = article_deletion_notifications.ArticleDeletionNotificationsBackgroundTask(job) + BackgroundApi.execute(task) + +if __name__ == "__main__": + main() diff --git a/portality/settings.py b/portality/settings.py index 0b0c4fef8c..3bc0c9030e 100644 --- a/portality/settings.py +++ b/portality/settings.py @@ -32,6 +32,9 @@ # CAUTION - this can modify the index so should NEVER be used in production! TESTDRIVE_ENABLED = False +# List of script names which can be executed via the testdrive. +TESTDRIVE_SCRIPT_WHITELIST = ["article_deletion_notifications"] + #################################### # Debug Mode @@ -451,6 +454,8 @@ "auto_assign_editor_group_data": {"month": "*", "day": "*/7", "day_of_week": "*", "hour": "3", "minute": "30"}, "ris_export": {"month": "*", "day": "15", "day_of_week": "*", "hour": "3", "minute": "30"}, "site_statistics": {"month": "*", "day": "*", "day_of_week": "*", "hour": "*", "minute": "40"}, + # Weekly notification to publishers about deleted articles (Article Tombstones) + "article_deletion_notifications": {"month": "*", "day": "*", "day_of_week": "1", "hour": "5", "minute": "10"}, } diff --git a/portality/tasks/article_deletion_notifications.py b/portality/tasks/article_deletion_notifications.py new file mode 100644 index 0000000000..9599246f68 --- /dev/null +++ b/portality/tasks/article_deletion_notifications.py @@ -0,0 +1,134 @@ +"""Weekly notifications to publishers about articles deleted in the last week. +Collect article details from ArticleTombstone and send a Notification per owner. +""" +from typing import Dict, List + +from portality import models, constants +from portality.background import BackgroundTask, BackgroundApi +from portality.bll.doaj import DOAJ +from portality.core import app +from portality.lib import dates +from portality.tasks.helpers import background_helper +from portality.tasks.redis_huey import scheduled_short_queue as queue +from portality.util import url_for + + +class ArticleDeletionNotificationsBackgroundTask(BackgroundTask): + __action__ = "article_deletion_notifications" + + def run(self): + job = self.background_job + + # note we're using the doaj url_for wrapper, not the flask url_for directly, due to the request context hack required + action_url = url_for("publisher.upload_file") + + # Query tombstones created in the last week + # Determine date range: last 7 days + since = dates.before_now(7 * 24 * 60 * 60) + q = models.ArticleTombstoneRecentlyDeletedQuery(since=since) + + owner_to_items: Dict[str, List[dict]] = {} + + for stone in models.ArticleTombstone.iterate_unstable(q.query(), page_size=1000): + owner = stone.owner + if not owner: + continue + + bj = stone.bibjson() + title = bj.title + authors = [] + for a in bj.author or []: + n = a.get('name') if isinstance(a, dict) else None + if n: + authors.append(n) + + volume = bj.volume + issue = bj.number + start_page = bj.start_page + end_page = bj.end_page + pages = None + if start_page and end_page: + pages = f"{start_page}-{end_page}" + elif start_page: + pages = str(start_page) + issns = bj.issns() + journal_name = bj.journal_title + + item = { + 'title': title, + 'authors': authors, + 'volume': volume, + 'issue': issue, + 'pages': pages, + 'issns': issns, + 'journal': journal_name, + } + + owner_to_items.setdefault(owner, []).append(item) + + if not owner_to_items: + job.add_audit_message("No deleted articles in the last week; no notifications sent.") + return + + notify_svc = DOAJ.notificationsService() + + total_notes = 0 + for owner, items in owner_to_items.items(): + acc = models.Account.pull(owner) + if acc is None: + job.add_audit_message(f"Owner account {owner} not found; skipping {len(items)} items") + continue + + source_id = "bg:job:" + self.__action__ + ":publisher" + + lines = [] + for it in items: + auth = (", ".join(it['authors'])) if it['authors'] else "Unknown author" + issn_str = ", ".join([i for i in (it.get('issns') or []) if i]) + vol_str = f", vol {it['volume']}" if it.get('volume') else "" + iss_str = f", issue {it['issue']}" if it.get('issue') else "" + pg_str = f", pages {it['pages']}" if it.get('pages') else "" + jn = f" ({it['journal']})" if it.get('journal') else "" + line = f"- {it['title'] or 'Untitled'} by {auth}{vol_str}{iss_str}{pg_str} [{issn_str}]{jn}" + lines.append(line) + + note = models.Notification() + note.who = owner + note.classification = constants.NOTIFICATION_CLASSIFICATION_STATUS + note.long = notify_svc.long_notification(source_id).format(list_of_articles="\n".join(lines)) + note.short = notify_svc.short_notification(source_id) + note.action = action_url + + notify_svc.notify(note) + total_notes += 1 + job.add_audit_message(f"Sending notification for owner {owner} with {len(items)} deleted articles") + + job.add_audit_message(f"Sent {total_notes} publisher deletion notifications.") + + def cleanup(self): + pass + + @classmethod + def prepare(cls, username, **kwargs): + job = background_helper.create_job(username, cls.__action__, + queue_id=huey_helper.queue_id) + return job + + @classmethod + def submit(cls, background_job): + background_job.save() + execute_article_deletion_notifications.schedule(args=(background_job.id,), + delay=app.config.get('HUEY_ASYNC_DELAY', 10)) + + +huey_helper = ArticleDeletionNotificationsBackgroundTask.create_huey_helper(queue) + + +@huey_helper.register_schedule +def scheduled_article_deletion_notifications(): + background_helper.submit_by_bg_task_type(ArticleDeletionNotificationsBackgroundTask) + + +@huey_helper.register_execute(is_load_config=False) +def execute_article_deletion_notifications(job_id): + background_helper.execute_by_job_id(job_id, lambda j: ArticleDeletionNotificationsBackgroundTask(j)) diff --git a/portality/tasks/consumer_scheduled_short_queue.py b/portality/tasks/consumer_scheduled_short_queue.py index daaa9917b2..a804373204 100644 --- a/portality/tasks/consumer_scheduled_short_queue.py +++ b/portality/tasks/consumer_scheduled_short_queue.py @@ -22,5 +22,6 @@ from portality.tasks.read_news import scheduled_read_news, read_news # noqa from portality.tasks.reporting import scheduled_reports, run_reports # noqa from portality.tasks.request_es_backup import scheduled_request_es_backup, request_es_backup # noqa +from portality.tasks.article_deletion_notifications import scheduled_article_deletion_notifications, execute_article_deletion_notifications #noqa from portality.tasks.auto_assign_editor_group_data import scheduled_auto_assign_editor_group_data, auto_assign_editor_group_data # noqa from portality.tasks.site_statistics import scheduled_site_statistics, site_statistics # noqa diff --git a/portality/templates-v2/dev/testdrive/testdrive.html b/portality/templates-v2/dev/testdrive/testdrive.html index 49770cb6a2..0d435fc5db 100644 --- a/portality/templates-v2/dev/testdrive/testdrive.html +++ b/portality/templates-v2/dev/testdrive/testdrive.html @@ -10,7 +10,7 @@