Add version metadata to objects (#745)

* add workflow metadata to objects

* rubocop fixes

* fix rabbitmq test

* switch to a new VersionMetadata model to make associations clearer

* allow workflow metadata to be deleted by passing in an empty hash

* remove dual primary key, and use index and validation instead

* move druid validation to a validator

* store all metadata in the database as json

* remove the parsing back and forth from json

* update comments

* parse JSON in controller; version now accepts hash

* rerun rubocop todo

* add info to README

* rename VersionMetadata to VersionContext; use POST for context instead of GET params

* serialize json in table; update tests

* client response should have json still for easier parsing by client

* remove unneeded code; make tests match how client works

Author: peetucket

.rubocop_todo.yml

@@ -1,12 +1,12 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2023-09-11 17:19:21 UTC using RuboCop version 1.56.3.
+# on 2024-04-18 19:41:40 UTC using RuboCop version 1.63.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 2
+# Offense count: 3
 # Configuration parameters: CountComments, CountAsOne, AllowedMethods, AllowedPatterns.
 Metrics/MethodLength:
   Max: 11
@@ -16,7 +16,7 @@ RSpec/AnyInstance:
   Exclude:
     - 'spec/requests/workflows/update_step_spec.rb'
 
-# Offense count: 19
+# Offense count: 28
 # Configuration parameters: CountAsOne.
 RSpec/ExampleLength:
   Max: 33
@@ -28,16 +28,16 @@ RSpec/LetSetup:
     - 'spec/services/sweeper_spec.rb'
     - 'spec/services/workflow_monitor_spec.rb'
 
-# Offense count: 46
+# Offense count: 55
 RSpec/MultipleExpectations:
   Max: 6
 
-# Offense count: 9
+# Offense count: 16
 # Configuration parameters: AllowSubject.
 RSpec/MultipleMemoizedHelpers:
   Max: 8
 
-# Offense count: 5
+# Offense count: 6
 # Configuration parameters: AllowedGroups.
 RSpec/NestedGroups:
   Max: 4
@@ -79,7 +79,7 @@ Style/SlicingWithRange:
   Exclude:
     - 'app/services/intersect_query.rb'
 
-# Offense count: 11
+# Offense count: 20
 # This cop supports safe autocorrection (--autocorrect).
 # Configuration parameters: AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns.
 # URISchemes: http, https

README.md

@@ -112,6 +112,24 @@ GET    /workflow_queue/all_queued
 GET    /workflow_queue
 ```
 
+### Workflow Variables
+
+If a workflow or workflows for a particular object require data to be persisted and available between steps, workflow variables can be set.
+These are per object/version pair and thus available to any step in any workflow for a given version of an object once set.
+
+These data are not persisted in Cocina, and are not preserved or available outside of the workflow-service, so they should only be used to persist information used during workflow processing.
+
+To use, pass in a "context" parameter as JSON in the body of the request when creating a workflow (and set content type to application/json). The json can contain any number of key/value pairs of context:
+
+```
+POST   /objects/:druid/workflows/:workflow?version=Y
+```
+
+This context will then be returned as JSON in each `process` block of the XML response containing workflow data, e.g. `GET    /objects/:druid/workflows` for use in processing.
+
+This can be used if a user selects an option in Pre-assembly or Argo that needs to be passed through the accessioning pipeline, such as if OCR or captioning is required.  The value is set when creating the workflow, and then available to each robot which needs it.
+
+
 ## Deploy
 ### Logs
 Logs are located in `/var/log/httpd`.

app/controllers/workflows_controller.rb

@@ -40,6 +40,7 @@ def show
     @workflow = Workflow.new(name: params[:workflow], druid: params[:druid], steps: workflow_steps)
   end
 
+  # rubocop:disable Metrics/AbcSize
   def create
     return render(plain: 'Unknown workflow', status: :bad_request) if template.nil?
 
@@ -48,12 +49,14 @@ def create
       processes: initial_parser.processes,
       version: Version.new(
         druid: params[:druid],
-        version: params[:version]
+        version: params[:version],
+        context: params[:context] # any context in the body of the request as JSON; wrapped in "context" key to allow for future body values
       )
     ).create_workflow_steps
 
     head :created
   end
+  # rubocop:enable Metrics/AbcSize
 
   def destroy
     obj = Version.new(

app/models/version.rb

@@ -1,14 +1,27 @@
 # frozen_string_literal: true
 
-# Represents a version of a digital object.
+# Represents a version of a digital object with associated context.
 # All workflow steps occur on to a particular version.
 class Version
-  def initialize(druid:, version:)
+  def initialize(druid:, version:, context: nil)
     @druid = druid
     @version_id = version
+    @context = context # this is context as a hash to be stored in the VersionContext table
   end
 
-  attr_reader :druid, :version_id
+  attr_reader :druid, :version_id, :context
+
+  def update_context
+    # if no context is passed in (nil), do nothing
+    return unless context
+
+    # if context is passed in but is empty, delete the version context record to clear all context
+    if context.blank?
+      VersionContext.find_by(druid:, version: version_id)&.destroy
+    else # otherwise, create/update the version context record as json in the database
+      VersionContext.find_or_create_by(druid:, version: version_id).update!(values: context)
+    end
+  end
 
   # @return [ActiveRecord::Relationship] an ActiveRecord scope that has the WorkflowSteps for this version
   def workflow_steps(workflow)

app/models/version_context.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+# Models optional context that is associated with a druid/version pair for any workflow
+class VersionContext < ApplicationRecord
+  validates :druid, uniqueness: { scope: :version }
+  validates_with DruidValidator
+end

app/models/workflow_step.rb

@@ -2,7 +2,7 @@
 
 # Models a process that occurred for a digital object. Basically a log entry.
 class WorkflowStep < ApplicationRecord
-  validate :druid_is_valid
+  validates_with DruidValidator
   validates :workflow, presence: true
   validates :process, presence: true
   validates :version, numericality: { only_integer: true }
@@ -42,6 +42,11 @@ def maybe_set_completed
     self.completed_at ||= Time.now
   end
 
+  # any associated version context for this step (if it exists) -- note: same for any workflow/step for a given druid/version combination
+  def context
+    VersionContext.find_by(druid:, version:)&.values
+  end
+
   ##
   # indicate if this step is marked as completed
   # @return [boolean]
@@ -59,13 +64,6 @@ def milestone_date
     end
   end
 
-  ##
-  # check if we have a valid druid with prefix
-  # @return [boolean]
-  def valid_druid?
-    DruidTools::Druid.valid?(druid, true) && druid.starts_with?('druid:')
-  end
-
   ##
   # check if the named workflow has a current definition
   # @return [boolean]
@@ -83,11 +81,6 @@ def valid_process_for_workflow?
     wtp.processes.map(&:name).include? process
   end
 
-  # ensure we have a valid druid with prefix
-  def druid_is_valid
-    errors.add(:druid, 'is not valid') unless valid_druid?
-  end
-
   # ensure we have a valid workflow before creating a new step
   def workflow_exists
     errors.add(:workflow, 'is not valid') unless valid_workflow?
@@ -99,6 +92,7 @@ def process_exists_for_workflow
   end
 
   # rubocop:disable Metrics/MethodLength
+  # rubocop:disable Metrics/AbcSize
   def attributes_for_process
     {
       version:,
@@ -108,11 +102,13 @@ def attributes_for_process
       elapsed:,
       attempts:,
       datetime: updated_at.to_time.iso8601,
+      context: context&.to_json, # context (which is deserialized as a hash by activerecord) as json so it can be deserialized by client
       status:,
       name: process
     }.tap do |attr|
       attr[:errorMessage] = error_msg if error_msg
     end
   end
   # rubocop:enable Metrics/MethodLength
+  # rubocop:enable Metrics/AbcSize
 end

app/services/workflow_creator.rb

@@ -18,6 +18,7 @@ def initialize(processes:, workflow_id:, version:)
   ##
   # Delete all the rows for this druid/version/workflow, and replace with new rows.
   # @return [Array]
+  # rubocop:disable Metrics/AbcSize
   def create_workflow_steps
     ActiveRecord::Base.transaction do
       version.workflow_steps(workflow_id).destroy_all
@@ -28,9 +29,13 @@ def create_workflow_steps
       processes.map do |process|
         WorkflowStep.create!(workflow_attributes(process))
       end
+
+      # Create/update version context
+      version.update_context
     end
     enqueue
   end
+  # rubocop:enable Metrics/AbcSize
 
   private
 

app/validators/druid_validator.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+# Validates the a druid is valid and starts with 'druid:'
+class DruidValidator < ActiveModel::Validator
+  def validate(record)
+    return if DruidTools::Druid.valid?(record.druid, true) && record.druid.starts_with?('druid:')
+
+    record.errors.add(:druid, 'is not valid')
+  end
+end

db/migrate/20240402194159_create_version_contexts.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+class CreateVersionContexts < ActiveRecord::Migration[7.0]
+  def change
+    create_table :version_contexts do |t|
+      t.string      :druid, null: false
+      t.integer     :version, null: false, default: 1
+      t.jsonb       :values, default: {}
+      t.timestamps
+    end
+
+    add_index :version_contexts, %i[druid version], unique: true
+  end
+end

db/schema.rb

@@ -10,11 +10,19 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2020_08_11_212454) do
-
+ActiveRecord::Schema[7.0].define(version: 2024_04_02_194159) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "version_contexts", force: :cascade do |t|
+    t.string "druid", null: false
+    t.integer "version", default: 1, null: false
+    t.jsonb "values", default: {}
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["druid", "version"], name: "index_version_contexts_on_druid_and_version", unique: true
+  end
+
   create_table "workflow_steps", id: :serial, force: :cascade do |t|
     t.string "druid", null: false
     t.string "workflow", null: false
@@ -28,10 +36,10 @@
     t.integer "version"
     t.text "note"
     t.string "lane_id", default: "default", null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
+    t.datetime "created_at", precision: nil, null: false
+    t.datetime "updated_at", precision: nil, null: false
     t.boolean "active_version", default: false
-    t.datetime "completed_at"
+    t.datetime "completed_at", precision: nil
     t.index ["active_version", "status", "workflow", "process"], name: "active_version_step_name_workflow2_idx"
     t.index ["druid", "version"], name: "index_workflow_steps_on_druid_and_version"
     t.index ["druid"], name: "index_workflow_steps_on_druid"

spec/factories/version_contexts.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :version_context do
+    sequence :druid do |n|
+      "druid:bb123bc#{format('%04d', n)}" # ensure we always have a valid druid format
+    end
+    version { 1 }
+    values { { requireOCR: true, requireTranscript: true } }
+  end
+end

spec/factories/workflow_steps.rb

@@ -3,7 +3,7 @@
 FactoryBot.define do
   factory :workflow_step do
     sequence :druid do |n|
-      "druid:bb123bc#{n.to_s.rjust(4, '0')}" # ensure we always have a valid druid format
+      "druid:bb123bc#{format('%04d', n)}" # ensure we always have a valid druid format
     end
     workflow { 'accessionWF' }
     process { 'start-accession' }
@@ -15,5 +15,11 @@
       status { 'completed' }
       completed_at { Time.now }
     end
+
+    trait :with_ocr_context do
+      after(:create) do |step|
+        create(:version_context, druid: step.druid, version: step.version)
+      end
+    end
   end
 end

spec/models/version_context_spec.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+RSpec.describe VersionContext do
+  let(:version_context) { FactoryBot.create(:version_context) }
+
+  it 'includes the context as a hash' do
+    expect(version_context.values).to eq({ 'requireOCR' => true, 'requireTranscript' => true })
+  end
+
+  it 'validates the uniqueness of druid and version combination' do
+    expect(described_class.new(druid: version_context.druid, version: version_context.version)).not_to be_valid
+  end
+
+  it 'validates the druid' do
+    expect(described_class.new(druid: 'foo', version: '1')).not_to be_valid
+  end
+end

spec/models/workflow_step_spec.rb

@@ -135,6 +135,20 @@
     end
   end
 
+  context 'with workflow context' do
+    let(:step_with_context) { FactoryBot.create(:workflow_step, :with_ocr_context) }
+
+    it 'includes the context as json' do
+      expect(step_with_context.context).to eq({ 'requireOCR' => true, 'requireTranscript' => true })
+    end
+  end
+
+  context 'without workflow context' do
+    it 'includes the context as nil' do
+      expect(step.context).to be_nil
+    end
+  end
+
   describe '#completed?' do
     it 'indicates if the step is not completed' do
       expect(step).not_to be_completed