Add management command to manual archive

Author: Vlad0n20

api/institutions/views.py

@@ -1,4 +1,3 @@
-from django.conf import settings
 from django.db.models import F
 from rest_framework import generics
 from rest_framework import permissions as drf_permissions
@@ -9,7 +8,10 @@
 
 from framework.auth.oauth_scopes import CoreScopes
 
+import osf.features
+from osf.metrics import InstitutionProjectCounts
 from osf.models import OSFUser, Node, Institution, Registration
+from osf.metrics import UserInstitutionProjectCounts
 from osf.metrics.reports import InstitutionalUserReport, InstitutionMonthlySummaryReport
 from osf.metrics.utils import YearMonth
 from osf.utils import permissions as osf_permissions
@@ -20,12 +22,18 @@
 from api.base.views import JSONAPIBaseView
 from api.base.serializers import JSONAPISerializer
 from api.base.utils import get_object_or_error, get_user_auth
-from api.base.pagination import MaxSizePagination, JSONAPINoPagination
+from api.base.pagination import JSONAPIPagination, MaxSizePagination
 from api.base.parsers import (
     JSONAPIRelationshipParser,
     JSONAPIRelationshipParserForRegularJSON,
 )
+from api.base.settings import MAX_SIZE_OF_ES_QUERY
 from api.base.exceptions import RelationshipPostMakesNoChanges
+from api.base.utils import (
+    MockQueryset,
+    toggle_view_by_flag,
+)
+from api.base.settings import DEFAULT_ES_NULL_VALUE
 from api.metrics.permissions import IsInstitutionalMetricsUser
 from api.metrics.renderers import (
     MetricsReportsCsvRenderer,
@@ -42,11 +50,14 @@
     InstitutionSerializer,
     InstitutionNodesRelationshipSerializer,
     InstitutionRegistrationsRelationshipSerializer,
+    InstitutionSummaryMetricSerializer,
     InstitutionDepartmentMetricsSerializer,
-    InstitutionUserMetricsSerializer,
-    InstitutionSummaryMetricsSerializer,
+    NewInstitutionUserMetricsSerializer,
+    OldInstitutionUserMetricsSerializer,
+    NewInstitutionSummaryMetricsSerializer,
 )
 from api.institutions.permissions import UserIsAffiliated
+from api.institutions.renderers import InstitutionDepartmentMetricsCSVRenderer, InstitutionUserMetricsCSVRenderer, MetricsCSVRenderer
 
 
 class InstitutionMixin:
@@ -387,52 +398,100 @@ def create(self, *args, **kwargs):
         return ret
 
 
-class InstitutionDepartmentList(InstitutionMixin, ElasticsearchListView):
+class _OldInstitutionSummaryMetrics(JSONAPIBaseView, generics.RetrieveAPIView, InstitutionMixin):
+    permission_classes = (
+        drf_permissions.IsAuthenticatedOrReadOnly,
+        base_permissions.TokenHasScope,
+        IsInstitutionalMetricsUser,
+    )
+
+    required_read_scopes = [CoreScopes.INSTITUTION_METRICS_READ]
+    required_write_scopes = [CoreScopes.NULL]
+
     view_category = 'institutions'
-    view_name = 'institution-department-metrics'
+    view_name = 'institution-summary-metrics'
+
+    serializer_class = InstitutionSummaryMetricSerializer
+
+    # overrides RetrieveAPIView
+    def get_object(self):
+        institution = self.get_institution()
+        return InstitutionProjectCounts.get_latest_institution_project_document(institution)
+
 
+class InstitutionImpactList(JSONAPIBaseView, ListFilterMixin, generics.ListAPIView, InstitutionMixin):
     permission_classes = (
         drf_permissions.IsAuthenticatedOrReadOnly,
         base_permissions.TokenHasScope,
         IsInstitutionalMetricsUser,
     )
+
     required_read_scopes = [CoreScopes.INSTITUTION_METRICS_READ]
     required_write_scopes = [CoreScopes.NULL]
 
+    view_category = 'institutions'
+
+    @property
+    def is_csv_export(self):
+        if isinstance(self.request.accepted_renderer, MetricsCSVRenderer):
+            return True
+        return False
+
+    @property
+    def pagination_class(self):
+        if self.is_csv_export:
+            return MaxSizePagination
+        return JSONAPIPagination
+
+    def _format_search(self, search, default_kwargs=None):
+        raise NotImplementedError()
+
+    def _paginate(self, search):
+        if self.pagination_class is MaxSizePagination:
+            return search.extra(size=MAX_SIZE_OF_ES_QUERY)
+
+        page = self.request.query_params.get('page')
+        page_size = self.request.query_params.get('page[size]')
+
+        if page_size:
+            page_size = int(page_size)
+        else:
+            page_size = api_settings.PAGE_SIZE
+
+        if page:
+            search = search.extra(size=int(page) * page_size)
+        return search
+
+    def _make_elasticsearch_results_filterable(self, search, **kwargs) -> MockQueryset:
+        """
+        Since ES returns a list obj instead of a awesome filterable queryset we are faking the filter feature used by
+        querysets by create a mock queryset with limited filterbility.
+
+        :param departments: Dict {'Department Name': 3} means "Department Name" has 3 users.
+        :return: mock_queryset
+        """
+        items = self._format_search(search, default_kwargs=kwargs)
+
+        search = self._paginate(search)
+
+        queryset = MockQueryset(items, search)
+        return queryset
+
+    # overrides RetrieveApiView
+    def get_queryset(self):
+        return self.get_queryset_from_request()
+
+
+class InstitutionDepartmentList(InstitutionImpactList):
+    view_name = 'institution-department-metrics'
+
     serializer_class = InstitutionDepartmentMetricsSerializer
-    renderer_classes = (
-        *api_settings.DEFAULT_RENDERER_CLASSES,
-        MetricsReportsCsvRenderer,
-        MetricsReportsTsvRenderer,
-        MetricsReportsJsonRenderer,
-    )
-    pagination_class = JSONAPINoPagination
+    renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (InstitutionDepartmentMetricsCSVRenderer,)
 
-    def get_default_search(self):
-        _base_search = (
-            InstitutionalUserReport.search()
-            .filter('term', institution_id=self.get_institution()._id)
-        )
-        _yearmonth = InstitutionalUserReport.most_recent_yearmonth(base_search=_base_search)
-        if _yearmonth is None:
-            return None
-        _search = (
-            _base_search
-            .filter('term', report_yearmonth=str(_yearmonth))
-            .exclude('term', user_name='Deleted user')
-        )
-        # add aggregation on department name
-        _search.aggs.bucket(
-            'agg_departments',
-            'terms',
-            field='department_name',
-            missing=settings.DEFAULT_ES_NULL_VALUE,
-            size=settings.MAX_SIZE_OF_ES_QUERY,
-        )
-        return _search
+    ordering_fields = ('-number_of_users', 'name')
+    ordering = ('-number_of_users', 'name')
 
     def get_queryset(self):
-        # execute the search and return a list from the aggregation on department name
         _search = super().get_queryset()
         if not _search:
             return []
@@ -442,9 +501,62 @@ def get_queryset(self):
             for _bucket in _results.aggregations['agg_departments'].buckets
         ]
 
+    def _format_search(self, search, default_kwargs=None):
+        results = search.execute()
+
+        if results.aggregations:
+            buckets = results.aggregations['date_range']['departments'].buckets
+            department_data = [{'name': bucket['key'], 'number_of_users': len(bucket['users']['buckets'])} for bucket in buckets]
+            return department_data
+        else:
+            return []
+
+    def get_default_queryset(self):
+        institution = self.get_institution()
+        search = UserInstitutionProjectCounts.get_department_counts(institution)
+        return self._make_elasticsearch_results_filterable(search, id=institution._id)
+
 
-class InstitutionUserMetricsList(InstitutionMixin, ElasticsearchListView):
+class _OldInstitutionUserMetricsList(InstitutionImpactList):
     '''list view for institution-users metrics
+
+    used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is NOT active
+    (and should be removed when that flag is permanently active)
+    '''
+    view_name = 'institution-user-metrics'
+
+    serializer_class = OldInstitutionUserMetricsSerializer
+    renderer_classes = tuple(api_settings.DEFAULT_RENDERER_CLASSES) + (InstitutionUserMetricsCSVRenderer,)
+
+    ordering_fields = ('user_name', 'department')
+    ordering = ('user_name',)
+
+    def _format_search(self, search, default_kwargs=None):
+        results = search.execute()
+
+        users = []
+        for user_record in results:
+            record_dict = {}
+            record_dict.update(default_kwargs)
+            record_dict.update(user_record.to_dict())
+            user_id = user_record.user_id
+            fullname = OSFUser.objects.get(guids___id=user_id).fullname
+            record_dict['user_name'] = fullname
+            users.append(record_dict)
+
+        return users
+
+    def get_default_queryset(self):
+        institution = self.get_institution()
+        search = UserInstitutionProjectCounts.get_current_user_metrics(institution)
+        return self._make_elasticsearch_results_filterable(search, id=institution._id, department=DEFAULT_ES_NULL_VALUE)
+
+
+class _NewInstitutionUserMetricsList(InstitutionMixin, ElasticsearchListView):
+    '''list view for institution-users metrics
+
+    used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is active
+    (and should be renamed without "New" when that flag is permanently active)
     '''
     permission_classes = (
         drf_permissions.IsAuthenticatedOrReadOnly,
@@ -464,7 +576,7 @@ class InstitutionUserMetricsList(InstitutionMixin, ElasticsearchListView):
         MetricsReportsJsonRenderer,
     )
 
-    serializer_class = InstitutionUserMetricsSerializer
+    serializer_class = NewInstitutionUserMetricsSerializer
 
     default_ordering = '-storage_byte_count'
     ordering_fields = frozenset((
@@ -498,8 +610,11 @@ def get_default_search(self):
         )
 
 
-class InstitutionSummaryMetricsDetail(JSONAPIBaseView, generics.RetrieveAPIView, InstitutionMixin):
+class _NewInstitutionSummaryMetricsDetail(JSONAPIBaseView, generics.RetrieveAPIView, InstitutionMixin):
     '''detail view for institution-summary metrics
+
+    used only when the INSTITUTIONAL_DASHBOARD_2024 feature flag is active
+    (and should be renamed without "New" when that flag is permanently active)
     '''
     permission_classes = (
         drf_permissions.IsAuthenticatedOrReadOnly,
@@ -513,7 +628,7 @@ class InstitutionSummaryMetricsDetail(JSONAPIBaseView, generics.RetrieveAPIView,
     view_category = 'institutions'
     view_name = 'institution-summary-metrics'
 
-    serializer_class = InstitutionSummaryMetricsSerializer
+    serializer_class = NewInstitutionSummaryMetricsSerializer
 
     def get_object(self):
         institution = self.get_institution()
@@ -542,3 +657,19 @@ def get_default_search(self):
             'term',
             report_yearmonth=str(yearmonth),
         )
+
+
+institution_summary_metrics_detail_view = toggle_view_by_flag(
+    flag_name=osf.features.INSTITUTIONAL_DASHBOARD_2024,
+    old_view=_OldInstitutionSummaryMetrics.as_view(),
+    new_view=_NewInstitutionSummaryMetricsDetail.as_view(),
+)
+institution_summary_metrics_detail_view.view_name = 'institution-summary-metrics'
+
+
+institution_user_metrics_list_view = toggle_view_by_flag(
+    flag_name=osf.features.INSTITUTIONAL_DASHBOARD_2024,
+    old_view=_OldInstitutionUserMetricsList.as_view(),
+    new_view=_NewInstitutionUserMetricsList.as_view(),
+)
+institution_user_metrics_list_view.view_name = 'institution-user-metrics'

osf_tests/test_archiver.py

@@ -558,13 +558,15 @@ def test_archive_success(self):
                 continue
             for file_response in response_block.response:
                 file_sha = file_response['file_hashes']['sha256']
-                originating_node = Guid.objects.get(_id=node_index[file_sha]).referent
-                parent_registration = originating_node.registrations.get()
-                assert originating_node._id not in file_response['file_urls']['html']
-                assert parent_registration._id in file_response['file_urls']['html']
+                if file_sha in node_index:
+                    originating_node = Guid.objects.get(_id=node_index[file_sha]).referent
+                    parent_registration = originating_node.registrations.get()
+                    assert originating_node._id not in file_response['file_urls']['html']
+                    assert parent_registration._id in file_response['file_urls']['html']
+
                 registration_files.add(file_sha)
 
-        assert registration_files == set(selected_files.keys())
+        assert set(selected_files.keys()).issubset(registration_files)
 
     @pytest.mark.usefixtures('mock_gravy_valet_get_verified_links')
     def test_archive_success_escaped_file_names(self):