Merge branch 'main' of github.com:CDLUC3/mrt-admin-lambda

Author: terrywbrady

src-admintool/config/reports.yml

@@ -376,6 +376,13 @@ audit_status:
     - For status `system-unavailable` or `unverified`, a retry is generally recommended.
     - A status of `size-mismatch` or `digest-mismatch` indicates that either a copy error occurred or this could show evidence of bit rot.  Immediate investigation is required.
     - Files with a status of `verified` are explicitly excluded from this report for performance reasons.
+audit_ucb:
+  link-title: Audit State - new UCB content
+  breadcrumb: bp_audit
+  class: AuditNewUCBQuery
+  category: audit
+  description: |
+    Look at audit status for new UCB content. 
 audit_status_time:
   link-title: Audit Status - Recent Processing
   breadcrumb: bp_audit

src-admintool/queries/audit_ucb_query.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+# Query class - see config/reports.yml for description
+class AuditNewUCBQuery < AdminQuery
+  def initialize(query_factory, path, myparams)
+    super
+    @days = get_param('days', 7).to_i
+    @mindays = 1
+  end
+
+  def get_title
+    "Audit Status for UCB Objects Modified in the Last #{@days} Days"
+  end
+
+  def get_sql
+    %{
+      select distinct
+        o.id,
+        o.ark,
+        o.modified,
+        count(a.id) as files_count,
+        max(a.verified) as verified,
+        a.status as audit_status,
+        case
+          when a.status in ('size-mismatch','digest-mismatch') then 'Audit Failed'
+          when ifnull(verified, o.modified) < date_add(o.modified, interval #{@mindays} day) and o.modified > date_add(now(), interval -#{@mindays} day) then 'Reset Later'
+          when a.status = 'verified' and ifnull(verified, o.modified) < date_add(o.modified, interval #{@mindays} day) then 'Reset Needed'
+          when a.status = 'verified' then 'Audited'
+          else 'In Progress'
+        end as category,
+        case
+          when a.status in ('size-mismatch','digest-mismatch') then 'FAIL'
+          when ifnull(verified, o.modified) < date_add(o.modified, interval #{@mindays} day) and o.modified > date_add(now(), interval -#{@mindays} day) then 'INFO'
+          when a.status = 'verified' and ifnull(verified, o.modified) < date_add(o.modified, interval #{@mindays} day) then 'WARN'
+          when a.status = 'verified' then 'PASS'
+          else 'INFO'
+        end as status
+      from 
+        inv.inv_objects o
+      right join
+        inv.inv_audits a 
+      on 
+        a.inv_object_id = o.id 
+      and 
+        a.inv_node_id = 16 /*sdsc node*/
+      where 
+        o.inv_owner_id=14 /*ucb owner*/
+      and 
+        o.modified > date_add(now(), interval -#{@days.to_i} day)
+      group by
+        o.id,
+        o.ark,
+        o.modified,
+        a.status
+      order by
+        o.modified desc
+      ;
+    }
+  end
+
+  def get_headers(_results)
+    ['Object Id', 'Ark', 'Obj Modified', 'Files', 'Verified', 'Audit Status', 'Category', 'Status']
+  end
+
+  def get_types(_results)
+    %w[objlist ark datetime dataint datetime '' '' status]
+  end
+
+  def get_alternative_queries
+    [
+      {
+        label: 'Last 7 days',
+        url: 'path=audit_ucb&days=7',
+        class: 'graph'
+      },
+      {
+        label: 'Last 14 days',
+        url: 'path=audit_ucb&days=14',
+        class: 'graph'
+      },
+      {
+        label: 'Last 30 days',
+        url: 'path=audit_ucb&days=30',
+        class: 'graph'
+      }
+    ]
+  end
+
+  def init_status
+    :PASS
+  end
+end

src-admintool/web/index.html

@@ -345,6 +345,8 @@ <h2>Audit Service</h2>
       <h3>Audit Status</h3>
       <ul>
         <li class="graph"><a href="{{ADMINTOOL_HOME}}?path=audit_status">Audit - Irregular Status</a></li>
+        <li class="graph"><a href="{{ADMINTOOL_HOME}}?path=audit_ucb">Audit - New UCB Content</a></li>
+        <li class="graph"><a href="{{COLLADMIN_HOME}}?path=audit_ucb_reset">Audit - Reset New UCB Content</a></li>
       </ul>
     </div>
 

src-colladmin/actions/ucb_audit_reset_action.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+
+require_relative 'action'
+
+# Collection Admin Task class - see config/actions.yml for description
+class UCBAuditResetAction < AdminAction
+  def get_title
+    "UCB Audit Reset"
+  end
+
+  def table_headers
+    %w[ObjectID Ark Modified Status]
+  end
+
+  def table_types
+    %w[objidlist ark datetime status]
+  end
+
+  def table_rows(_body)
+    sql = %{
+      select distinct
+        o.id, o.ark, o.modified, 'PASS' as status
+      from 
+        inv.inv_objects o
+      inner join
+        inv.inv_audits a 
+      on 
+        a.inv_object_id = o.id 
+      and 
+        a.inv_node_id = 16 /*sdsc node*/
+      where 
+        o.inv_owner_id=14 /*ucb owner*/
+      and 
+        o.modified > date_add(now(), interval -7 day)
+      and
+        ifnull(verified, o.modified) < date_add(o.modified, interval 1 day)
+      and 
+        o.modified < date_add(now(), interval -1 day)
+      and 
+        a.status = 'verified'
+      order by o.modified
+      limit 2
+      ;
+    }
+    data = MerrittQuery.new(@config).run_query(sql)
+    data.each do |r|
+      objid = r[0]
+      MerrittQuery.new(@config).run_update(
+        %(
+          update
+            inv_audits
+          set
+            verified = null,
+            status = 'unknown'
+          where
+            inv_object_id = ?
+          and
+            inv_node_id = ?
+        ),
+        [objid, 16],
+        'Audit reset for object'
+      )
+    end
+    data
+  end
+
+  def perform_action
+    convert_json_to_table({}.to_json)
+  end
+
+  def has_table
+    true
+  end
+
+  def init_status
+    :PASS
+  end
+
+end

src-colladmin/config/actions.yml

@@ -681,6 +681,14 @@ storage-clear-audit-batch:
     This function clears all audit batches older than a specified amount of time.
   documentation: |
     SQL: UPDATE inv_audits SET status='unknown', verified=null
+audit_ucb_reset:
+  link-title: Reset Audit Status for new UCB content
+  class: UCBAuditResetAction
+  category: Audit
+  sensitivity: reversible change
+  testing: manual
+  description: |
+    After 1 (1 or 2) days, trigger an immediate re-audit of new UCB content.
 storage-retry-audit-status:
   link-title: Trigger an audit retry for all audit records with a specific status
   class: StorageAction

src-colladmin/lambda_function.rb

@@ -30,6 +30,7 @@
 require_relative 'actions/consistency_report_cleanup_action'
 require_relative 'actions/lambda_tag_action'
 require_relative 'actions/system_state_action'
+require_relative 'actions/ucb_audit_reset_action'
 require 'yaml'
 
 # Handle GET or POST event structures pass in via the ALB