Refactor the XLSX report logic into a single function #1524

For re-usability: `get_xlsx_report`

Signed-off-by: tdruez <[email protected]>

Author: tdruez

scanpipe/forms.py

@@ -297,15 +297,15 @@ class ProjectReportForm(BaseProjectActionForm):
     model_name = forms.ChoiceField(
         label="Choose the object type to include in the XLSX file",
         choices=[
-            ("discoveredpackage", "Packages"),
-            ("discovereddependency", "Dependencies"),
-            ("codebaseresource", "Resources"),
-            ("codebaserelation", "Relations"),
-            ("projectmessage", "Messages"),
+            ("package", "Packages"),
+            ("dependency", "Dependencies"),
+            ("resource", "Resources"),
+            ("relation", "Relations"),
+            ("message", "Messages"),
             ("todo", "TODOs"),
         ],
         required=True,
-        initial="discoveredpackage",
+        initial="package",
         widget=forms.RadioSelect,
     )
 

scanpipe/management/commands/report.py

@@ -26,8 +26,6 @@
 from django.core.management import CommandError
 from django.core.management.base import BaseCommand
 
-import xlsxwriter
-
 from aboutcode.pipeline import humanize_time
 from scanpipe.models import Project
 from scanpipe.pipes import filename_now
@@ -76,7 +74,6 @@ def handle(self, *args, **options):
         labels = options["labels"]
         search = options["search"]
         sheet = options["sheet"]
-        model_name = output.object_type_to_model_name.get(sheet)
 
         if not (labels or search):
             raise CommandError(
@@ -97,23 +94,13 @@ def handle(self, *args, **options):
             msg = f"{project_count} project(s) will be included in the report."
             self.stdout.write(msg, self.style.SUCCESS)
 
-        worksheet_queryset = output.get_queryset(project=None, model_name=model_name)
-        worksheet_queryset = worksheet_queryset.filter(project__in=project_qs)
-
         filename = f"scancodeio-report-{filename_now()}.xlsx"
         if output_directory:
             output_file = Path(f"{output_directory}/{filename}")
         else:
             output_file = Path(filename)
 
-        with xlsxwriter.Workbook(output_file) as workbook:
-            output.queryset_to_xlsx_worksheet(
-                worksheet_queryset,
-                workbook,
-                exclude_fields=output.XLSX_EXCLUDE_FIELDS,
-                prepend_fields=["project"],
-                worksheet_name="TODOS",
-            )
+        output_file = output.get_xlsx_report(project_qs, sheet, output_file)
 
         run_time = timer() - start_time
         if self.verbosity > 0:

scanpipe/pipes/output.py

@@ -22,6 +22,7 @@
 
 import csv
 import decimal
+import io
 import json
 import re
 from operator import attrgetter
@@ -301,6 +302,7 @@ def to_json(project):
     "codebaseresource": "RESOURCES",
     "codebaserelation": "RELATIONS",
     "projectmessage": "MESSAGES",
+    "todo": "TODOS",
 }
 
 model_name_to_object_type = {
@@ -399,6 +401,32 @@ def add_xlsx_worksheet(workbook, worksheet_name, rows, fields):
     return errors_count
 
 
+# TODO: Add unit test
+def get_xlsx_report(project_qs, model_short_name, output_file=None):
+    model_name = object_type_to_model_name.get(model_short_name)
+    if not model_name:
+        raise ValueError
+
+    worksheet_name = model_name_to_worksheet_name.get(model_short_name)
+
+    worksheet_queryset = get_queryset(project=None, model_name=model_name)
+    worksheet_queryset = worksheet_queryset.filter(project__in=project_qs)
+
+    if not output_file:
+        output_file = io.BytesIO()
+
+    with xlsxwriter.Workbook(output_file) as workbook:
+        queryset_to_xlsx_worksheet(
+            worksheet_queryset,
+            workbook,
+            exclude_fields=XLSX_EXCLUDE_FIELDS,
+            prepend_fields=["project"],
+            worksheet_name=worksheet_name,
+        )
+
+    return output_file
+
+
 # Some scan attributes such as "copyrights" are list of dicts.
 #
 #  'authors': [{'end_line': 7, 'start_line': 7, 'author': 'John Doe'}],

scanpipe/tests/test_views.py

@@ -20,6 +20,7 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
+import io
 import json
 import shutil
 import uuid
@@ -34,6 +35,7 @@
 from django.urls import reverse
 from django.urls.exceptions import NoReverseMatch
 
+import openpyxl
 import requests
 
 from scanpipe.forms import BaseProjectActionForm
@@ -228,7 +230,12 @@ def test_scanpipe_views_project_action_report_view(self):
             "model_name": "todo",
         }
         response = self.client.post(url, data=data, follow=True)
-        self.assertEqual("report.xlsx", response.filename)
+        self.assertTrue(response.filename.startswith("scancodeio-report-"))
+        self.assertTrue(response.filename.endswith(".xlsx"))
+
+        output_file = io.BytesIO(b"".join(response.streaming_content))
+        workbook = openpyxl.load_workbook(output_file, read_only=True, data_only=True)
+        self.assertEqual(["TODOS"], workbook.get_sheet_names())
 
     def test_scanpipe_views_project_action_view_get_project_queryset(self):
         queryset = ProjectActionView.get_project_queryset(

scanpipe/views.py

@@ -96,6 +96,7 @@
 from scanpipe.models import RunInProgressError
 from scanpipe.pipes import compliance
 from scanpipe.pipes import count_group_by
+from scanpipe.pipes import filename_now
 from scanpipe.pipes import output
 from scanpipe.pipes import purldb
 
@@ -453,24 +454,14 @@ def get_export_xlsx_queryset(self):
     def get_export_xlsx_filename(self):
         return f"{self.project.name}_{self.model._meta.model_name}.xlsx"
 
-    def get_export_xlsx_prepend_fields(self):
-        return []
-
-    def get_export_xlsx_worksheet_name(self):
-        return
-
     def export_xlsx_file_response(self):
         output_file = io.BytesIO()
         queryset = self.get_export_xlsx_queryset()
-        prepend_fields = self.get_export_xlsx_prepend_fields()
-        worksheet_name = self.get_export_xlsx_worksheet_name()
         with xlsxwriter.Workbook(output_file) as workbook:
             output.queryset_to_xlsx_worksheet(
                 queryset,
                 workbook,
                 exclude_fields=output.XLSX_EXCLUDE_FIELDS,
-                prepend_fields=prepend_fields,
-                worksheet_name=worksheet_name,
             )
 
         output_file.seek(0)
@@ -1191,7 +1182,7 @@ def form_valid(self, form):
 
 
 @method_decorator(require_POST, name="dispatch")
-class ProjectActionView(ConditionalLoginRequired, ExportXLSXMixin, generic.ListView):
+class ProjectActionView(ConditionalLoginRequired, generic.ListView):
     """Call a method for each instance of the selection."""
 
     model = Project
@@ -1222,9 +1213,7 @@ def post(self, request, *args, **kwargs):
             return self.download_outputs_zip_response(project_qs, action_form)
 
         if action == "report":
-            self.action_form = action_form
-            self.project_qs = project_qs
-            return self.export_xlsx_file_response()
+            return self.xlsx_report_response(project_qs, action_form)
 
         if action == "archive":
             action_kwargs = action_form.get_action_kwargs()
@@ -1287,21 +1276,6 @@ def get_project_queryset(selected_project_ids=None, action_form=None):
 
         raise Http404
 
-    def get_export_xlsx_queryset(self):
-        model_name = self.action_form.cleaned_data["model_name"]
-        queryset = output.get_queryset(project=None, model_name=model_name)
-        return queryset.filter(project__in=self.project_qs)
-
-    def get_export_xlsx_prepend_fields(self):
-        return ["project"]
-
-    def get_export_xlsx_worksheet_name(self):
-        if self.action_form.cleaned_data.get("model_name") == "todo":
-            return "TODOS"
-
-    def get_export_xlsx_filename(self):
-        return "report.xlsx"
-
     @staticmethod
     def download_outputs_zip_response(project_qs, action_form):
         output_format = action_form.cleaned_data["output_format"]
@@ -1323,6 +1297,18 @@ def download_outputs_zip_response(project_qs, action_form):
             filename="scancodeio_output_files.zip",
         )
 
+    @staticmethod
+    def xlsx_report_response(project_qs, action_form):
+        model_short_name = action_form.cleaned_data["model_name"]
+        filename = f"scancodeio-report-{filename_now()}.xlsx"
+        output_file = output.get_xlsx_report(project_qs, model_short_name)
+        output_file.seek(0)
+        return FileResponse(
+            output_file,
+            as_attachment=True,
+            filename=filename,
+        )
+
 
 class HTTPResponseHXRedirect(HttpResponseRedirect):
     status_code = 200