Merge branch 'main' into production

Author: mouse-reeve

.env.example

@@ -137,3 +137,6 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60
 # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default.
 # Value should be a comma-separated list of host names.
 CSP_ADDITIONAL_HOSTS=
+# The last number here means "megabytes"
+# Increase if users are having trouble uploading BookWyrm export files.
+DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 100)

bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.23 on 2024-01-16 10:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("bookwyrm", "0191_merge_20240102_0326"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="sitesettings",
+            name="user_exports_enabled",
+            field=models.BooleanField(default=False),
+        ),
+    ]

bookwyrm/models/site.py

@@ -96,6 +96,7 @@ class SiteSettings(SiteModel):
     imports_enabled = models.BooleanField(default=True)
     import_size_limit = models.IntegerField(default=0)
     import_limit_reset = models.IntegerField(default=0)
+    user_exports_enabled = models.BooleanField(default=False)
     user_import_time_limit = models.IntegerField(default=48)
 
     field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"])

bookwyrm/settings.py

@@ -442,3 +442,5 @@
 # Do not change this setting unless you already have an existing
 # user with the same username - in which case you should change it!
 INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor"
+
+DATA_UPLOAD_MAX_MEMORY_SIZE = env.int("DATA_UPLOAD_MAX_MEMORY_SIZE", (1024**2 * 100))

bookwyrm/templates/moved.html

@@ -23,7 +23,7 @@
 
           <div class="notification is-warning">
             <p>
-                {% id_to_username request.user.moved_to  as username %}
+                {% id_to_username request.user.moved_to as username %}
                 {% blocktrans trimmed with moved_to=user.moved_to %}
                     <strong>You have moved your account</strong> to <a href="{{ moved_to }}">{{ username }}</a>
                 {% endblocktrans %}

bookwyrm/templates/notifications/items/move_user.html

@@ -14,7 +14,7 @@
 
 {% block description %}
     {% if related_user_moved_to %}
-        {% id_to_username request.user.moved_to  as username %}
+        {% id_to_username related_user_moved_to as username %}
         {% blocktrans trimmed %}
         {{ related_user }} has moved to <a href="{{ related_user_moved_to }}">{{ username }}</a>
         {% endblocktrans %}

bookwyrm/templates/preferences/export-user.html

@@ -46,7 +46,11 @@ <h2 class="is-size-5">Your file will not include:</h2>
             {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an <strong>alias</strong> of this one, or <strong>move</strong> this account to the new account, before you import your user data." %}
             {% endspaceless %}
         </p>
-    {% if next_available %}
+    {% if not site.user_exports_enabled %}
+    <p class="notification is-danger">
+        {% trans "New user exports are currently disabled." %}
+    </p>
+    {% elif next_available %}
     <p class="notification is-warning">
         {% blocktrans trimmed %}
         You will be able to create a new export file at {{ next_available }}

bookwyrm/templates/settings/imports/imports.html

@@ -90,6 +90,33 @@
             </div>
         </form>
     </details>
+
+    {% if site.user_exports_enabled %}
+    <details class="details-panel box">
+        <summary>
+            <span role="heading" aria-level="2" class="title is-6">
+                {% trans "Disable starting new user exports" %}
+            </span>
+            <span class="details-close icon icon-x" aria-hidden="true"></span>
+        </summary>
+        <form
+            name="disable-user-exports"
+            id="disable-user-exports"
+            method="POST"
+            action="{% url 'settings-user-exports-disable' %}"
+        >
+            <div class="notification">
+                {% trans "This is only intended to be used when things have gone very wrong with exports and you need to pause the feature while addressing issues." %}
+                {% trans "While exports are disabled, users will not be allowed to start new user exports, but existing exports will not be affected." %}
+            </div>
+            {% csrf_token %}
+            <div class="control">
+                <button type="submit" class="button is-danger">
+                    {% trans "Disable user exports" %}
+                </button>
+            </div>
+        </form>
+    </details>
     <details class="details-panel box">
         <summary>
             <span role="heading" aria-level="2" class="title is-6">
@@ -108,7 +135,7 @@
                 {% trans "Set the value to 0 to not enforce any limit." %}
             </div>
             <div class="align.to-t">
-                <label for="limit">{% trans "Restrict user imports and exports to once every " %}</label>
+                <label for="limit">{% trans "Limit how often users can import and export user data" %}</label>
                 <input name="limit" class="input is-w-xs is-h-em" type="text" placeholder="0" value="{{ user_import_time_limit }}">
                 <label>{% trans "hours" %}</label>
                 {% csrf_token %}
@@ -120,6 +147,28 @@
             </div>
         </form>
     </details>
+    {% else %}
+    <form
+        name="enable-user-imports"
+        id="enable-user-imports"
+        method="POST"
+        action="{% url 'settings-user-exports-enable' %}"
+        class="box"
+    >
+        <div class="notification is-danger is-light">
+            <p class="my-2">{% trans "Users are currently unable to start new user exports. This is the default setting." %}</p>
+            {% if use_s3 %}
+            <p>{% trans "It is not currently possible to provide user exports when using s3 storage. The BookWyrm development team are working on a fix for this." %}</p>
+            {% endif %}
+        </div>
+        {% csrf_token %}
+        <div class="control">
+            <button type="submit" class="button is-success" {% if use_s3 %}disabled{% endif %}>
+                {% trans "Enable user exports" %}
+            </button>
+        </div>
+    </form>
+    {% endif %}
 </div>
 <div class="block">
     <h4 class="title is-4">{% trans "Book Imports" %}</h4>

bookwyrm/templatetags/utilities.py

@@ -125,7 +125,8 @@ def id_to_username(user_id):
         name = parts[-1]
         value = f"{name}@{domain}"
 
-    return value
+        return value
+    return "a new user account"
 
 
 @register.filter(name="get_file_size")

bookwyrm/tests/views/preferences/test_export.py

@@ -18,7 +18,9 @@ class ExportViews(TestCase):
     """viewing and creating statuses"""
 
     @classmethod
-    def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
+    def setUpTestData(
+        self,
+    ):  # pylint: disable=bad-classmethod-argument, disable=invalid-name
         """we need basic test data and mocks"""
         with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
             "bookwyrm.activitystreams.populate_stream_task.delay"
@@ -40,6 +42,7 @@ def setUpTestData(self):  # pylint: disable=bad-classmethod-argument
             bnf_id="beep",
         )
 
+    # pylint: disable=invalid-name
     def setUp(self):
         """individual test setup"""
         self.factory = RequestFactory()
@@ -53,11 +56,12 @@ def tst_export_get(self, *_):
 
     def test_export_file(self, *_):
         """simple export"""
-        models.ShelfBook.objects.create(
+        shelfbook = models.ShelfBook.objects.create(
             shelf=self.local_user.shelf_set.first(),
             user=self.local_user,
             book=self.book,
         )
+        book_date = str.encode(f"{shelfbook.shelved_date.date()}")
         request = self.factory.post("")
         request.user = self.local_user
         export = views.Export.as_view()(request)
@@ -66,7 +70,7 @@ def test_export_file(self, *_):
         # pylint: disable=line-too-long
         self.assertEqual(
             export.content,
-            b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,rating,review_name,review_cw,review_content\r\nTest Book,,"
-            + self.book.remote_id.encode("utf-8")
-            + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,\r\n",
+            b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\n"
+            + b"Test Book,,%b,,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,to-read,To Read,%b\r\n"
+            % (self.book.remote_id.encode("utf-8"), book_date),
         )

bookwyrm/urls.py

@@ -338,6 +338,16 @@
         views.disable_imports,
         name="settings-imports-disable",
     ),
+    re_path(
+        r"^settings/user-exports/enable/?$",
+        views.enable_user_exports,
+        name="settings-user-exports-enable",
+    ),
+    re_path(
+        r"^settings/user-exports/disable/?$",
+        views.disable_user_exports,
+        name="settings-user-exports-disable",
+    ),
     re_path(
         r"^settings/imports/enable/?$",
         views.enable_imports,

bookwyrm/views/__init__.py

@@ -18,6 +18,8 @@
     set_import_size_limit,
     set_user_import_completed,
     set_user_import_limit,
+    enable_user_exports,
+    disable_user_exports,
 )
 from .admin.ip_blocklist import IPBlocklist
 from .admin.invite import ManageInvites, Invite, InviteRequest

bookwyrm/views/admin/imports.py

@@ -9,7 +9,7 @@
 
 from bookwyrm import models
 from bookwyrm.views.helpers import redirect_to_referer
-from bookwyrm.settings import PAGE_LENGTH
+from bookwyrm.settings import PAGE_LENGTH, USE_S3
 
 
 # pylint: disable=no-self-use
@@ -59,6 +59,7 @@ def get(self, request, status="active"):
             "import_size_limit": site_settings.import_size_limit,
             "import_limit_reset": site_settings.import_limit_reset,
             "user_import_time_limit": site_settings.user_import_time_limit,
+            "use_s3": USE_S3,
         }
         return TemplateResponse(request, "settings/imports/imports.html", data)
 
@@ -126,3 +127,25 @@ def set_user_import_limit(request):
     site.user_import_time_limit = int(request.POST.get("limit"))
     site.save(update_fields=["user_import_time_limit"])
     return redirect("settings-imports")
+
+
+@require_POST
+@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
+# pylint: disable=unused-argument
+def enable_user_exports(request):
+    """Allow users to export account data"""
+    site = models.SiteSettings.objects.get()
+    site.user_exports_enabled = True
+    site.save(update_fields=["user_exports_enabled"])
+    return redirect("settings-imports")
+
+
+@require_POST
+@permission_required("bookwyrm.edit_instance_settings", raise_exception=True)
+# pylint: disable=unused-argument
+def disable_user_exports(request):
+    """Don't allow users to export account data"""
+    site = models.SiteSettings.objects.get()
+    site.user_exports_enabled = False
+    site.save(update_fields=["user_exports_enabled"])
+    return redirect("settings-imports")

bookwyrm/views/preferences/export.py

@@ -17,7 +17,8 @@
 from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob
 from bookwyrm.settings import PAGE_LENGTH
 
-# pylint: disable=no-self-use
+
+# pylint: disable=no-self-use,too-many-locals
 @method_decorator(login_required, name="dispatch")
 class Export(View):
     """Let users export data"""
@@ -54,7 +55,19 @@ def post(self, request):
         fields = (
             ["title", "author_text"]
             + deduplication_fields
-            + ["rating", "review_name", "review_cw", "review_content"]
+            + [
+                "start_date",
+                "finish_date",
+                "stopped_date",
+                "rating",
+                "review_name",
+                "review_cw",
+                "review_content",
+                "review_published",
+                "shelf",
+                "shelf_name",
+                "shelf_date",
+            ]
         )
         writer.writerow(fields)
 
@@ -70,6 +83,24 @@ def post(self, request):
 
             book.rating = review_rating.rating if review_rating else None
 
+            readthrough = (
+                models.ReadThrough.objects.filter(user=request.user, book=book)
+                .order_by("-start_date", "-finish_date")
+                .first()
+            )
+            if readthrough:
+                book.start_date = (
+                    readthrough.start_date.date() if readthrough.start_date else None
+                )
+                book.finish_date = (
+                    readthrough.finish_date.date() if readthrough.finish_date else None
+                )
+                book.stopped_date = (
+                    readthrough.stopped_date.date()
+                    if readthrough.stopped_date
+                    else None
+                )
+
             review = (
                 models.Review.objects.filter(
                     user=request.user, book=book, content__isnull=False
@@ -78,9 +109,27 @@ def post(self, request):
                 .first()
             )
             if review:
+                book.review_published = (
+                    review.published_date.date() if review.published_date else None
+                )
                 book.review_name = review.name
                 book.review_cw = review.content_warning
-                book.review_content = review.raw_content
+                book.review_content = (
+                    review.raw_content if review.raw_content else review.content
+                )  # GoodReads imported reviews do not have raw_content, but content.
+
+            shelfbook = (
+                models.ShelfBook.objects.filter(user=request.user, book=book)
+                .order_by("-shelved_date", "-created_date", "-updated_date")
+                .last()
+            )
+            if shelfbook:
+                book.shelf = shelfbook.shelf.identifier
+                book.shelf_name = shelfbook.shelf.name
+                book.shelf_date = (
+                    shelfbook.shelved_date.date() if shelfbook.shelved_date else None
+                )
+
             writer.writerow([getattr(book, field, "") or "" for field in fields])
 
         return HttpResponse(

nginx/development

@@ -64,13 +64,18 @@ server {
     # directly serve images and static files from the
     # bookwyrm filesystem using sendfile.
     # make the logs quieter by not reporting these requests
-    location ~ ^/(images|static)/ {
+    location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ {
         root /app;
         try_files $uri =404;
         add_header X-Cache-Status STATIC;
         access_log off;
     }
 
+    # block access to any non-image files from images or static
+    location ~ ^/images/ {
+        return 403;
+    }
+
     # monitor the celery queues with flower, no caching enabled
     location /flower/ {
        proxy_pass http://flower:8888;

nginx/production

@@ -96,12 +96,17 @@ server {
 #     # directly serve images and static files from the
 #     # bookwyrm filesystem using sendfile.
 #     # make the logs quieter by not reporting these requests
-#     location ~ ^/(images|static)/ {
+#     location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ {
 #         root /app;
 #         try_files $uri =404;
 #         add_header X-Cache-Status STATIC;
 #         access_log off;
 #     }
+
+#     # block access to any non-image files from images or static
+#     location ~ ^/images/ {
+#          return 403;
+#     }
 #
 #     # monitor the celery queues with flower, no caching enabled
 #     location /flower/ {