refactor various places where os.path is used in various CSV related …

…places + tests

kolibri/core/auth/csv_utils.py

@@ -176,7 +176,7 @@ def csv_file_generator(facility, filename, overwrite=True, demographic=False):
         column for column in db_columns if demographic or column not in DEMO_FIELDS
     )
 
-    csv_file = io.StringIO() 
+    csv_file = io.StringIO()
 
     mappings = {}
 

kolibri/core/auth/management/commands/bulkexportusers.py

@@ -2,7 +2,6 @@
 import io
 import logging
 import ntpath
-import os
 from collections import OrderedDict
 from functools import partial
 
@@ -29,7 +28,6 @@
 from kolibri.core.tasks.management.commands.base import AsyncCommand
 from kolibri.core.tasks.utils import get_current_job
 from kolibri.core.utils.csv import output_mapper
-from kolibri.utils import conf
 
 try:
     FileNotFoundError
@@ -266,16 +264,14 @@ def get_facility(self, options):
         return default_facility
 
     def get_filepath(self, options, facility):
+        file_storage = DefaultStorage()
         if options["output_file"] is None:
-            export_dir = os.path.join(conf.KOLIBRI_HOME, "log_export")
-            if not os.path.isdir(export_dir):
-                os.mkdir(export_dir)
-            filepath = os.path.join(
-                export_dir,
+            filename = (
                 CSV_EXPORT_FILENAMES["user"].format(facility.name, facility.id[:4]),
             )
+            filepath = file_storage.path(filename)
         else:
-            filepath = os.path.join(os.getcwd(), options["output_file"])
+            filepath = file_storage.path(options["output_file"])
         return filepath
 
     def handle_async(self, *args, **options):

kolibri/core/logger/csv_export.py

@@ -240,7 +240,7 @@ def map_object(item, topic_headers_length):
 
 
 def csv_file_generator(
-    facility, log_type, filepath, start_date, end_date, overwrite=False
+    facility, log_type, filename, start_date, end_date, overwrite=False
 ):
     file_storage = DefaultStorage()
 
@@ -257,7 +257,6 @@ def csv_file_generator(
         else parser.parse(end_date) + datetime.timedelta(days=1)
     )
 
-    filename = file_storage.generate_filename(filepath.split("/")[-1])
     if not overwrite and file_storage.exists(filename):
         raise ValueError("{} already exists".format(filename))
     queryset = log_info["queryset"].filter(

kolibri/core/logger/management/commands/exportlogs.py

@@ -1,6 +1,4 @@
 import logging
-import ntpath
-import os
 
 from dateutil import parser
 from django.conf import settings
@@ -128,8 +126,6 @@ def handle_async(self, *args, **options):
             else:
                 filename = options["output_file"]
 
-            filepath = os.path.join(os.getcwd(), filename)
-
             queryset = log_info["queryset"]
 
             total_rows = queryset.count()
@@ -139,7 +135,7 @@ def handle_async(self, *args, **options):
                     for row in csv_file_generator(
                         facility,
                         log_type,
-                        filepath,
+                        filename,
                         start_date=start_date,
                         end_date=end_date,
                         overwrite=options["overwrite"],
@@ -150,14 +146,14 @@ def handle_async(self, *args, **options):
 
         if job:
             job.extra_metadata["overall_error"] = self.overall_error
-            self.job.extra_metadata["filename"] = ntpath.basename(filepath)
+            self.job.extra_metadata["filename"] = filename
             job.save_meta()
         else:
             if self.overall_error:
                 raise CommandError(self.overall_error)
             else:
                 logger.info(
-                    "Created csv file {} with {} lines".format(filepath, total_rows)
+                    "Created csv file {} with {} lines".format(filename, total_rows)
                 )
 
         translation.deactivate()

kolibri/core/logger/tasks.py

@@ -1,5 +1,6 @@
 import os
 
+from django.core.files.storage import DefaultStorage
 from django.core.management import call_command
 from rest_framework import serializers
 
@@ -19,16 +20,20 @@
 
 def get_filepath(log_type, facility_id, start_date, end_date):
     facility = Facility.objects.get(id=facility_id)
-    logs_dir = os.path.join(conf.KOLIBRI_HOME, "log_export")
-    if not os.path.isdir(logs_dir):
-        os.mkdir(logs_dir)
-    filepath = os.path.join(
-        logs_dir,
-        CSV_EXPORT_FILENAMES[log_type].format(
-            facility.name, facility.id[:4], start_date[:10], end_date[:10]
-        ),
-    )
-    return filepath
+    file_storage = DefaultStorage()
+
+    for log_type in CSV_EXPORT_FILENAMES:
+        if log_type in ("summary", "session"):
+            log_path = file_storage.path(
+                CSV_EXPORT_FILENAMES[log_type].format(
+                    facility.name, facility.id[:4], start_date[:10], end_date[:10]
+                ),
+            )
+        else:
+            log_path = file_storage.path(
+                CSV_EXPORT_FILENAMES[log_type].format(facility.name, facility.id[:4])
+            )
+    return log_path
 
 
 def get_valid_logs_csv_filenames():

kolibri/plugins/facility/test/test_api.py

@@ -4,7 +4,6 @@
 Also tests whether the users with permissions can create logs.
 """
 import datetime
-import os
 import uuid
 
 import mock
@@ -21,29 +20,18 @@
 from kolibri.core.logger.test.factory_logger import ContentSummaryLogFactory
 from kolibri.core.logger.test.factory_logger import FacilityUserFactory
 from kolibri.plugins.facility.views import CSV_EXPORT_FILENAMES
-from kolibri.utils import conf
 from kolibri.utils.time_utils import utc_now
 
 
 def output_filename(log_type, facility, **kwargs):
-    logs_dir = os.path.join(conf.KOLIBRI_HOME, "log_export")
-    if not os.path.isdir(logs_dir):
-        os.mkdir(logs_dir)
     if log_type in ("summary", "session"):
         start_date = kwargs.get("start_date")
         end_date = kwargs.get("end_date")
-        log_path = os.path.join(
-            logs_dir,
-            CSV_EXPORT_FILENAMES[log_type].format(
-                facility.name, facility.id[:4], start_date[:10], end_date[:10]
-            ),
+        return CSV_EXPORT_FILENAMES[log_type].format(
+            facility.name, facility.id[:4], start_date[:10], end_date[:10]
         )
     else:
-        log_path = os.path.join(
-            logs_dir,
-            CSV_EXPORT_FILENAMES[log_type].format(facility.name, facility.id[:4]),
-        )
-    return log_path
+        return (CSV_EXPORT_FILENAMES[log_type].format(facility.name, facility.id[:4]),)
 
 
 class ContentSummaryLogCSVExportTestCase(APITestCase):

kolibri/plugins/facility/views.py

@@ -1,5 +1,4 @@
 import json
-import os
 from datetime import datetime as dt
 
 from django.core.exceptions import PermissionDenied
@@ -27,7 +26,6 @@
 )
 from kolibri.core.logger.models import ContentSessionLog
 from kolibri.core.logger.models import GenerateCSVLogRequest
-from kolibri.utils import conf
 
 
 CSV_EXPORT_FILENAMES = {}
@@ -95,9 +93,9 @@ def exported_csv_info(request, facility_id):
 
     :returns: An object with the files informatin
     """
+    file_storage = DefaultStorage()
     facility = _get_facility_check_permissions(request, facility_id)
 
-    logs_dir = os.path.join(conf.KOLIBRI_HOME, "log_export")
     csv_statuses = {}
 
     for log_type in CSV_EXPORT_FILENAMES:
@@ -109,19 +107,17 @@ def exported_csv_info(request, facility_id):
             else:
                 start = ""
                 end = ""
-            log_path = os.path.join(
-                logs_dir,
+            log_path = file_storage.path(
                 CSV_EXPORT_FILENAMES[log_type].format(
                     facility.name, facility.id[:4], start[:10], end[:10]
                 ),
             )
         else:
-            log_path = os.path.join(
-                logs_dir,
-                CSV_EXPORT_FILENAMES[log_type].format(facility.name, facility.id[:4]),
+            log_path = file_storage.path(
+                CSV_EXPORT_FILENAMES[log_type].format(facility.name, facility.id[:4])
             )
-        if os.path.exists(log_path):
-            csv_statuses[log_type] = os.path.getmtime(log_path)
+        if file_storage.exists(log_path):
+            csv_statuses[log_type] = file_storage.get_modified_time(log_path)
         else:
             csv_statuses[log_type] = None