feat: parallel sidekiq jobs

.github/workflows/push.yml

@@ -27,8 +27,11 @@ env:
   DF_ENCRYPTION_DETERMINISTIC_KEY: "anlmuJ6cB3bN3biXRbYvmPsC5ALPFqGG"
   DF_ENCRYPTION_KEY_DERIVATION_SALT: "hzPR8D4qpOnAg7VeAhkhWw6JmmzKJB10"
   DF_REDIS_SIDEKIQ_URL: "redis://redis:6379/0"
+  DF_API_CONTAINER_NAME: doubtfire-api-development
   LATEX_CONTAINER_NAME: doubtfire-texlive
+  DF_LATEX_IMAGE_NAME: doubtfire-texlive
   LATEX_BUILD_PATH: /texlive/shell/latex_build.sh
+  DF_JPLAG_IMAGE_NAME: doubtfire-jplag
   DF_JPLAG_REPORT_DIR: /jplag/results
   LTI_SHARED_API_SECRET: "abc123"
   LTI_ENABLED: true
@@ -61,7 +64,9 @@ jobs:
           file: texlive.Dockerfile
           push: false
           load: true
-          tags: doubtfire-texlive-development:local
+          tags: |
+            ${{ env.DF_LATEX_IMAGE_NAME }}:local
+            ${{ env.DF_LATEX_IMAGE_NAME }}:latest
           cache-from: type=gha,scope=texlive
           cache-to: type=gha,mode=max,scope=texlive
       - name: Build JPlag image
@@ -71,7 +76,9 @@ jobs:
           file: jplag.Dockerfile
           push: false
           load: true
-          tags: doubtfire-jplag-development:local
+          tags: |
+            ${{ env.DF_JPLAG_IMAGE_NAME }}:local
+            ${{ env.DF_JPLAG_IMAGE_NAME }}:latest
           cache-from: type=gha,scope=jplag
           cache-to: type=gha,mode=max,scope=jplag
       - name: Build base doubtfire-api development image
@@ -80,13 +87,13 @@ jobs:
           context: .
           push: false
           load: true
-          tags: doubtfire-api-development:local
+          tags: ${{ env.DF_API_CONTAINER_NAME }}:local
           cache-from: type=gha,scope=doubtfire-api
           cache-to: type=gha,mode=max,scope=doubtfire-api
       - name: Start TexLive service
         uses: addnab/docker-run-action@v3
         with:
-          image: doubtfire-texlive-development:local
+          image: ${{ env.DF_LATEX_IMAGE_NAME }}:local
           options: >
             --name ${{ env.LATEX_CONTAINER_NAME }}
             -v ${{ github.workspace }}/student-work:/student-work
@@ -98,7 +105,7 @@ jobs:
       - name: Test TexLive container
         uses: addnab/docker-run-action@v3
         with:
-          image: doubtfire-api-development:local
+          image: ${{ env.DF_API_CONTAINER_NAME }}:local
           options: >
             -t
             -v ${{ github.workspace }}:/doubtfire
@@ -107,9 +114,10 @@ jobs:
       - name: Start JPlag service
         uses: addnab/docker-run-action@v3
         with:
-          image: doubtfire-jplag-development:local
+          image: ${{ env.DF_JPLAG_IMAGE_NAME }}:local
           options: >
             --name jplag
+            -v ${{ github.workspace }}:/doubtfire
             -v ${{ github.workspace }}/student-work:/student-work
             -v ${{ github.workspace }}/jplag/results:${{ env.DF_JPLAG_REPORT_DIR }}
             -v ${{ github.workspace }}/tmp/jplag:/tmp/jplag
@@ -119,7 +127,7 @@ jobs:
       - name: Test JPlag service
         uses: addnab/docker-run-action@v3
         with:
-          image: doubtfire-api-development:local
+          image: ${{ env.DF_API_CONTAINER_NAME }}:local
           options: >
             -t
             -v ${{ github.workspace }}:/doubtfire
@@ -128,7 +136,7 @@ jobs:
       - name: Populate database
         uses: addnab/docker-run-action@v3
         with:
-          image: doubtfire-api-development:local
+          image: ${{ env.DF_API_CONTAINER_NAME }}:local
           options: >
             -v ${{ github.workspace }}:/doubtfire
             -v ${{ github.workspace }}/student-work:/student-work
@@ -153,14 +161,19 @@ jobs:
             -e LATEX_CONTAINER_NAME
             -e LATEX_BUILD_PATH
             -e DF_JPLAG_REPORT_DIR
+            -e DF_API_CONTAINER_NAME
+            -e DF_LATEX_IMAGE_NAME
+            -e DF_JPLAG_IMAGE_NAME
             -e LTI_SHARED_API_SECRET
             -e LTI_ENABLED
+            -e DF_LATEX_PATH_TO_WORKDIRS=/doubtfire/tmp/rails-latex
           run: bundle exec rake db:populate
       - name: Run unit tests
         uses: addnab/docker-run-action@v3
         with:
-          image: doubtfire-api-development:local
+          image: ${{ env.DF_API_CONTAINER_NAME }}:local
           options: >
+            --name ${{ env.DF_API_CONTAINER_NAME }}
             -v ${{ github.workspace }}:/doubtfire
             -v ${{ github.workspace }}/student-work:/student-work
             -v /var/run/docker.sock:/var/run/docker.sock
@@ -186,8 +199,13 @@ jobs:
             -e LATEX_CONTAINER_NAME
             -e LATEX_BUILD_PATH
             -e DF_JPLAG_REPORT_DIR
+            -e DF_API_CONTAINER_NAME
+            -e DF_LATEX_IMAGE_NAME
+            -e DF_JPLAG_IMAGE_NAME
             -e LTI_SHARED_API_SECRET
             -e LTI_ENABLED
+            -e DF_LATEX_PATH_TO_WORKDIRS=/doubtfire/tmp/rails-latex
+            -e DF_JPLAG_PATH_TO_WORKDIRS=/doubtfire/tmp/jplag
           run: TERM=xterm bundle exec rails test
       - name: Stop TexLive service
         run: docker rm -f ${{ env.LATEX_CONTAINER_NAME }}

app/helpers/file_helper.rb

@@ -409,35 +409,11 @@ def qpdf(path)
   # - retain_from = true if you want to keep from, otherwise it is deleted
   # - only_before = date for files to move (only if retain from is true)
   def move_files(from_path, to_path, retain_from = false, only_before = nil)
-    # move into the new dir - and mv files to the in_process_dir
-    begin
-      pwd = FileUtils.pwd
-    rescue
-      # if no pwd, reset to the root
-      pwd = Rails.root
-    end
-
-    begin
-      FileUtils.mkdir_p(to_path)
-      Dir.chdir(from_path)
-      FileUtils.mv Dir.glob('*').filter{|fn| !retain_from || only_before.nil? || File.ctime(fn) < only_before}, to_path, force: true
-      Dir.chdir(to_path)
-      begin
-        # remove from_path as files are now "in process"
-        # these can be retained when the old folder wants to be kept
-        FileUtils.rm_rf(from_path) unless retain_from
-      rescue
-        logger.warn "failed to rm #{from_path}"
-      end
-    ensure
-      if FileUtils.pwd != pwd
-        if Dir.exist? pwd
-          FileUtils.chdir(pwd)
-        else
-          FileUtils.chdir(student_work_dir)
-        end
-      end
-    end
+    FileUtils.mkdir_p(to_path)
+    files = Dir.glob(File.join(from_path, '*'))
+               .filter { |fn| !retain_from || only_before.nil? || File.ctime(fn) < only_before }
+    FileUtils.mv(files, to_path, force: true)
+    FileUtils.rm_rf(from_path) unless retain_from
   end
 
   #

app/models/pdf_generation/project_compile_portfolio_module.rb

@@ -65,6 +65,7 @@ def init(project, is_retry)
         @institution_name = Doubtfire::Application.config.institution[:name]
         @doubtfire_product_name = Doubtfire::Application.config.institution[:product_name]
         @is_retry = is_retry
+        @work_id = "portfolio-#{project.id}#{'-retry' if is_retry}"
       end
 
       def make_pdf
@@ -214,9 +215,8 @@ def move_to_portfolio(file, name, kind)
         self.uses_draft_learning_summary = false
         save
       else
-        Dir.chdir(portfolio_tmp_dir)
-        files = Dir.glob('*')
-        idx = files.map { |a_file| a_file.split('-').first.to_i }.max
+        files = Dir.glob(File.join(portfolio_tmp_dir, '*'))
+        idx = files.map { |a_file| File.basename(a_file).split('-').first.to_i }.max
         if idx.nil? || idx < 1
           idx = 1
         else
@@ -237,10 +237,11 @@ def portfolio_files(ensure_valid: false, force_ascii: false)
 
       result = []
 
-      Dir.chdir(portfolio_tmp_dir)
-      files = Dir.glob('*').select { |f| (f =~ /^\d{3}-(cover|document|code|image)/) == 0 }
+      files = Dir.glob(File.join(portfolio_tmp_dir, '*'))
+                 .select { |f| File.basename(f) =~ /^\d{3}-(cover|document|code|image)/ }
+
       files.each do |file|
-        parts = file.split('-')
+        parts = File.basename(file).split('-')
         idx = parts[0].to_i
         kind = parts[1]
         name = parts.drop(2).join('-')