1524 report api (#1559)

* Use new tab for download and report action #1524 Signed-off-by: tdruez <[email protected]> * Refactor the XLSX report logic into a single function #1524 For re-usability: `get_xlsx_report` Signed-off-by: tdruez <[email protected]> * Add unit test for get_xlsx_report #1524 Signed-off-by: tdruez <[email protected]> * Rename the --sheet option to --model #1524 Signed-off-by: tdruez <[email protected]> * Add support for the XLSX report in REST API #1524 Signed-off-by: tdruez <[email protected]> --------- Signed-off-by: tdruez <[email protected]>
aboutcode-org · Jan 22, 2025 · 84f4f99 · 84f4f99
1 parent 0297b40
commit 84f4f99
Show file tree

Hide file tree

Showing 14 changed files with 281 additions and 78 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -4,6 +4,9 @@ Changelog
 v34.9.5 (unreleased)
 --------------------
 
+- Add support for the XLSX report in REST API.
+  https://github.com/aboutcode-org/scancode.io/issues/1524
+
 v34.9.4 (2025-01-21)
 --------------------
 

diff --git a/docs/command-line-interface.rst b/docs/command-line-interface.rst
@@ -396,15 +396,15 @@ your outputs on the host machine when running with Docker.
 
 .. _cli_report:
 
-`$ scanpipe report --sheet SHEET`
+`$ scanpipe report --model MODEL`
 ---------------------------------
 
 Generates an XLSX report of selected projects based on the provided criteria.
 
 Required arguments:
 
-- ``--sheet {package,dependency,resource,relation,message,todo}``
-  Specifies the sheet to include in the XLSX report. Available choices are based on
+- ``--model {package,dependency,resource,relation,message,todo}``
+  Specifies the model to include in the XLSX report. Available choices are based on
   predefined object types.
 
 Optional arguments:
@@ -428,12 +428,12 @@ Example usage:
 1. Generate a report for all projects tagged with "d2d" and include the **TODOS**
 worksheet::
 
-   $ scanpipe report --sheet todo --label d2d
+   $ scanpipe report --model todo --label d2d
 
 2. Generate a report for projects whose names contain the word "audit" and include the
 **PACKAGES** worksheet::
 
-   $ scanpipe report --sheet package --search audit
+   $ scanpipe report --model package --search audit
 
 .. _cli_check_compliance:
 

diff --git a/docs/rest-api.rst b/docs/rest-api.rst
@@ -587,3 +587,69 @@ This action deletes a "not started" or "queued" pipeline run.
     {
         "status": "Pipeline pipeline_name deleted."
     }
+
+XLSX Report
+-----------
+
+Generates an XLSX report of selected projects based on the provided criteria.
+The model needs to be provided using the ``model`` query parameter.
+
+``GET /api/projects/?model=MODEL``
+
+Data:
+  - ``model``: ``package``, ``dependency``, ``resource``, ``relation``, ``message``,
+    ``todo``.
+
+**Any available filters can be applied** to **select the set of projects** you want to
+include in the report, such as a string contained in the name, or filter by labels:
+
+Example usage:
+
+1. Generate a report for all projects tagged with "d2d" and include the **TODOS**
+worksheet::
+
+    GET /api/projects/?model=todo&label=d2d
+
+
+2. Generate a report for projects whose names contain the word "audit" and include the
+**PACKAGES** worksheet::
+
+   GET /api/projects/?model=package&name__contains=audit
+
+
+XLSX Report
+-----------
+
+Generates an XLSX report for selected projects based on specified criteria. The
+``model`` query parameter is required to determine the type of data to include in the
+report.
+
+Endpoint:
+``GET /api/projects/report/?model=MODEL``
+
+Parameters:
+
+- ``model``: Defines the type of data to include in the report.
+  Accepted values: ``package``, ``dependency``, ``resource``, ``relation``, ``message``,
+  ``todo``.
+
+.. note::
+
+   You can apply any available filters to select the projects to include in the
+   report. Filters can be based on project attributes, such as a substring in the
+   name or specific labels.
+
+Example Usage:
+
+1. Generate a report for projects tagged with "d2d" and include the ``TODOS`` worksheet:
+
+   .. code-block::
+
+      GET /api/projects/report/?model=todo&label=d2d
+
+2. Generate a report for projects whose names contain "audit" and include the
+   ``PACKAGES`` worksheet:
+
+   .. code-block::
+
+      GET /api/projects/report/?model=package&name__contains=audit
diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py
@@ -52,6 +52,7 @@
 from scanpipe.models import Project
 from scanpipe.models import Run
 from scanpipe.models import RunInProgressError
+from scanpipe.pipes import filename_now
 from scanpipe.pipes import output
 from scanpipe.pipes.compliance import get_project_compliance_alerts
 from scanpipe.views import project_results_json_response
@@ -79,6 +80,11 @@ class ProjectFilterSet(django_filters.rest_framework.FilterSet):
         method="filter_names",
     )
     uuid = django_filters.CharFilter()
+    label = django_filters.CharFilter(
+        label="Label",
+        field_name="labels__slug",
+        distinct=True,
+    )
 
     class Meta:
         model = Project
@@ -90,6 +96,7 @@ class Meta:
             "names",
             "uuid",
             "is_archived",
+            "label",
         ]
 
     def filter_names(self, qs, name, value):
@@ -195,6 +202,40 @@ def pipelines(self, request, *args, **kwargs):
         ]
         return Response(pipeline_data)
 
+    @action(detail=False)
+    def report(self, request, *args, **kwargs):
+        project_qs = self.filter_queryset(self.get_queryset())
+
+        model_choices = list(output.object_type_to_model_name.keys())
+        model = request.GET.get("model")
+        if not model:
+            message = {
+                "error": (
+                    "Specifies the model to include in the XLSX report. "
+                    "Using: ?model=MODEL"
+                ),
+                "choices": ", ".join(model_choices),
+            }
+            return Response(message, status=status.HTTP_400_BAD_REQUEST)
+
+        if model not in model_choices:
+            message = {
+                "error": f"{model} is not on of the valid choices",
+                "choices": ", ".join(model_choices),
+            }
+            return Response(message, status=status.HTTP_400_BAD_REQUEST)
+
+        output_file = output.get_xlsx_report(
+            project_qs=project_qs,
+            model_short_name=model,
+        )
+        output_file.seek(0)
+        return FileResponse(
+            output_file,
+            filename=f"scancodeio-report-{filename_now()}.xlsx",
+            as_attachment=True,
+        )
+
     def get_filtered_response(
         self, request, queryset, filterset_class, serializer_class
     ):

diff --git a/scanpipe/forms.py b/scanpipe/forms.py
@@ -297,15 +297,15 @@ class ProjectReportForm(BaseProjectActionForm):
     model_name = forms.ChoiceField(
         label="Choose the object type to include in the XLSX file",
         choices=[
-            ("discoveredpackage", "Packages"),
-            ("discovereddependency", "Dependencies"),
-            ("codebaseresource", "Resources"),
-            ("codebaserelation", "Relations"),
-            ("projectmessage", "Messages"),
+            ("package", "Packages"),
+            ("dependency", "Dependencies"),
+            ("resource", "Resources"),
+            ("relation", "Relations"),
+            ("message", "Messages"),
             ("todo", "TODOs"),
         ],
         required=True,
-        initial="discoveredpackage",
+        initial="package",
         widget=forms.RadioSelect,
     )
 

diff --git a/scanpipe/management/commands/report.py b/scanpipe/management/commands/report.py
@@ -26,8 +26,6 @@
 from django.core.management import CommandError
 from django.core.management.base import BaseCommand
 
-import xlsxwriter
-
 from aboutcode.pipeline import humanize_time
 from scanpipe.models import Project
 from scanpipe.pipes import filename_now
@@ -48,10 +46,10 @@ def add_arguments(self, parser):
             ),
         )
         parser.add_argument(
-            "--sheet",
+            "--model",
             required=True,
             choices=list(output.object_type_to_model_name.keys()),
-            help="Specifies the sheet to include in the XLSX report.",
+            help="Specifies the model to include in the XLSX report.",
         )
         parser.add_argument(
             "--search",
@@ -75,8 +73,7 @@ def handle(self, *args, **options):
         output_directory = options["output_directory"]
         labels = options["labels"]
         search = options["search"]
-        sheet = options["sheet"]
-        model_name = output.object_type_to_model_name.get(sheet)
+        model = options["model"]
 
         if not (labels or search):
             raise CommandError(
@@ -97,23 +94,13 @@ def handle(self, *args, **options):
             msg = f"{project_count} project(s) will be included in the report."
             self.stdout.write(msg, self.style.SUCCESS)
 
-        worksheet_queryset = output.get_queryset(project=None, model_name=model_name)
-        worksheet_queryset = worksheet_queryset.filter(project__in=project_qs)
-
         filename = f"scancodeio-report-{filename_now()}.xlsx"
         if output_directory:
             output_file = Path(f"{output_directory}/{filename}")
         else:
             output_file = Path(filename)
 
-        with xlsxwriter.Workbook(output_file) as workbook:
-            output.queryset_to_xlsx_worksheet(
-                worksheet_queryset,
-                workbook,
-                exclude_fields=output.XLSX_EXCLUDE_FIELDS,
-                prepend_fields=["project"],
-                worksheet_name="TODOS",
-            )
+        output_file = output.get_xlsx_report(project_qs, model, output_file)
 
         run_time = timer() - start_time
         if self.verbosity > 0:

diff --git a/scanpipe/pipes/output.py b/scanpipe/pipes/output.py
@@ -22,6 +22,7 @@
 
 import csv
 import decimal
+import io
 import json
 import re
 from operator import attrgetter
@@ -301,6 +302,7 @@ def to_json(project):
     "codebaseresource": "RESOURCES",
     "codebaserelation": "RELATIONS",
     "projectmessage": "MESSAGES",
+    "todo": "TODOS",
 }
 
 model_name_to_object_type = {
@@ -399,6 +401,31 @@ def add_xlsx_worksheet(workbook, worksheet_name, rows, fields):
     return errors_count
 
 
+def get_xlsx_report(project_qs, model_short_name, output_file=None):
+    model_name = object_type_to_model_name.get(model_short_name)
+    if not model_name:
+        raise ValueError(f"{model_short_name} is not valid.")
+
+    worksheet_name = model_name_to_worksheet_name.get(model_short_name)
+
+    worksheet_queryset = get_queryset(project=None, model_name=model_name)
+    worksheet_queryset = worksheet_queryset.filter(project__in=project_qs)
+
+    if not output_file:
+        output_file = io.BytesIO()
+
+    with xlsxwriter.Workbook(output_file) as workbook:
+        queryset_to_xlsx_worksheet(
+            worksheet_queryset,
+            workbook,
+            exclude_fields=XLSX_EXCLUDE_FIELDS,
+            prepend_fields=["project"],
+            worksheet_name=worksheet_name,
+        )
+
+    return output_file
+
+
 # Some scan attributes such as "copyrights" are list of dicts.
 #
 #  'authors': [{'end_line': 7, 'start_line': 7, 'author': 'John Doe'}],

diff --git a/scanpipe/templates/scanpipe/modals/projects_download_modal.html b/scanpipe/templates/scanpipe/modals/projects_download_modal.html
@@ -6,7 +6,7 @@
       <p class="modal-card-title">Download outputs for selected projects as ZIP file</p>
       <button class="delete" aria-label="close"></button>
     </header>
-    <form action="{% url 'project_action' %}" method="post" id="download-projects-form">{% csrf_token %}
+    <form action="{% url 'project_action' %}" method="post" id="download-projects-form" target="_blank">{% csrf_token %}
       <section class="modal-card-body">
         <div class="field">
           <label class="label">{{ outputs_download_form.output_format.label }}</label>

diff --git a/scanpipe/templates/scanpipe/modals/projects_report_modal.html b/scanpipe/templates/scanpipe/modals/projects_report_modal.html
@@ -6,7 +6,7 @@
       <p class="modal-card-title">Report of selected projects</p>
       <button class="delete" aria-label="close"></button>
     </header>
-    <form action="{% url 'project_action' %}" method="post" id="report-projects-form">{% csrf_token %}
+    <form action="{% url 'project_action' %}" method="post" id="report-projects-form" target="_blank">{% csrf_token %}
       <section class="modal-card-body">
         <div class="field">
           <label class="label">{{ report_form.model_name.label }}</label>