Merge pull request #4409 from GSA-TTS/main

.github/workflows/deploy-application.yml

@@ -126,3 +126,11 @@ jobs:
           cf_space: ${{ env.space }}
           command: cf restart logshipper
 
+  check-tables:
+    needs: [push-with-creds]
+    uses: ./.github/workflows/fac-check-tables.yml
+    secrets: inherit
+    with:
+      environment: ${{ inputs.environment }}
+      util_version: "v0.1.8"
+      backup_operation: "check_tables"

.github/workflows/fac-backup-scheduler.yml

@@ -19,6 +19,6 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ matrix.environment.name }}
-      util_version: "v0.1.5"
+      util_version: "v0.1.8"
       backup_operation: "scheduled_backup"
 

.github/workflows/fac-backup-util-scheduled.yml

@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.5 scheduled_backup
-# ./fac-backup-util.sh v0.1.5 daily_backup
+# ./fac-backup-util.sh v0.1.8 scheduled_backup
+# ./fac-backup-util.sh v0.1.8 daily_backup
 on:
   workflow_call:
     inputs:

.github/workflows/fac-backup-util.yml

@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.5 initial_backup
-# ./fac-backup-util.sh v0.1.5 deploy_backup
+# ./fac-backup-util.sh v0.1.8 initial_backup
+# ./fac-backup-util.sh v0.1.8 deploy_backup
 on:
   workflow_dispatch:
     inputs:

.github/workflows/fac-check-tables-scheduler.yml

@@ -0,0 +1,24 @@
+---
+name: Check Tables Daily
+on:
+  schedule:
+    # Invoke every 12 hours
+    - cron: '0 */12 * * *'
+  workflow_dispatch: null
+
+jobs:
+  check-tables:
+    strategy:
+      fail-fast: false
+      matrix:
+        environment:
+          - name: dev
+          - name: staging
+          - name: production
+          - name: preview
+    uses: ./.github/workflows/fac-check-tables.yml
+    secrets: inherit
+    with:
+      environment: ${{ matrix.environment.name }}
+      util_version: "v0.1.8"
+      backup_operation: "check_tables"

.github/workflows/fac-check-tables.yml

@@ -0,0 +1,54 @@
+---
+name: Check existing tables in an environment
+### Common Commands:
+# ./fac-backup-util.sh v0.1.8 check_tables
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        options:
+          - 'dev'
+          - 'preview'
+          - 'staging'
+          - 'production'
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: choice
+        options:
+          - 'check_tables'
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: string
+jobs:
+  fac-check-tables:
+    name: Check tables in FAC Database
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+    steps:
+      - name: Check tables in ${{ env.space }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 2G -m 3G --name check_tables --command "./fac-backup-util.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"

.github/workflows/testing-from-build.yml

@@ -47,12 +47,25 @@ jobs:
           pwd
           ls -al | grep 'coverage'
 
-      - name: Coverage Action
+      - name: Code Coverage Summary Report
+        uses: irongut/[email protected]
+        with:
+          filename: ./coverage.xml
+          badge: true
+          fail_below_min: true
+          format: markdown
+          hide_branch_rate: false
+          hide_complexity: true
+          indicators: true
+          output: both
+          thresholds: '85 90'
+
+      - name: Add Coverage PR Comment
+        uses: marocchino/sticky-pull-request-comment@v2
         if: github.event_name == 'pull_request'
-        uses: orgoro/[email protected]
         with:
-            coverageFile: ./coverage.xml
-            token: ${{ secrets.GITHUB_TOKEN }}
+          recreate: true
+          path: code-coverage-results.md
 
   a11y-testing:
     runs-on: ubuntu-latest

.github/workflows/testing-from-ghcr.yml

@@ -49,12 +49,25 @@ jobs:
           pwd
           ls -al | grep 'coverage'
 
-      - name: Coverage Action
+      - name: Code Coverage Summary Report
+        uses: irongut/[email protected]
+        with:
+          filename: ./coverage.xml
+          badge: true
+          fail_below_min: false
+          format: markdown
+          hide_branch_rate: false
+          hide_complexity: true
+          indicators: true
+          output: both
+          thresholds: '85'
+
+      - name: Add Coverage PR Comment
+        uses: marocchino/sticky-pull-request-comment@v2
         if: github.event_name == 'pull_request'
-        uses: orgoro/[email protected]
         with:
-            coverageFile: ./coverage.xml
-            token: ${{ secrets.GITHUB_TOKEN }}
+          recreate: true
+          path: code-coverage-results.md
 
   a11y-testing:
     runs-on: ubuntu-latest

.github/workflows/trivy.yml

@@ -39,7 +39,7 @@ jobs:
         run: docker build -t ${{ env.DOCKER_NAME }}:${{ steps.date.outputs.date }} .
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.27.0
+        uses: aquasecurity/trivy-action@0.28.0
         with:
           image-ref: '${{ env.DOCKER_NAME }}:${{ steps.date.outputs.date }}'
           scan-type: 'image'
@@ -74,7 +74,7 @@ jobs:
         run: docker pull ${{ matrix.image.name }}
 
       - name: Run Trivy vulnerability scanner on Third Party Images
-        uses: aquasecurity/trivy-action@0.27.0
+        uses: aquasecurity/trivy-action@0.28.0
         with:
           image-ref: '${{ matrix.image.name }}'
           scan-type: 'image'

.github/workflows/zap-scan.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: ZAP Scan of ${{ env.url }}
-        uses: zaproxy/action-baseline@v0.12.0
+        uses: zaproxy/action-baseline@v0.13.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           docker_name: 'ghcr.io/zaproxy/zaproxy:stable'

backend/audit/intake_to_dissemination.py

@@ -246,7 +246,14 @@ def load_passthrough(self):
         self.loaded_objects["Passthroughs"] = pass_objects
         return pass_objects
 
-    def _get_dates_from_sac(self):
+    def _get_first_date_by_status_from_sac(self, status):
+        sac = self.single_audit_checklist
+        for i in range(len(sac.transition_name)):
+            if sac.transition_name[i] == status:
+                return sac.transition_date[i]
+        raise ValueError("This SAC does not have the requested status.")
+
+    def _get_most_recent_dates_from_sac(self):
         return_dict = dict()
         sac = self.single_audit_checklist
         for status_choice in sac.STATUS_CHOICES:
@@ -283,12 +290,12 @@ def load_general(self):
         cognizant_agency = self.single_audit_checklist.cognizant_agency
         oversight_agency = self.single_audit_checklist.oversight_agency
 
-        dates_by_status = self._get_dates_from_sac()
+        dates_by_status = self._get_most_recent_dates_from_sac()
         status = self.single_audit_checklist.get_statuses()
         ready_for_certification_date = dates_by_status[status.READY_FOR_CERTIFICATION]
         if self.mode == IntakeToDissemination.DISSEMINATION:
             submitted_date = self._convert_utc_to_american_samoa_zone(
-                dates_by_status[status.SUBMITTED]
+                self._get_first_date_by_status_from_sac(status.SUBMITTED)
             )
             fac_accepted_date = submitted_date
             auditee_certify_name = auditee_certification["auditee_signature"][

backend/audit/intakelib/checks/runners.py

@@ -1,7 +1,7 @@
 from django.core.exceptions import ValidationError
 import logging
 
-from .check_finding_uniqueness import check_finding_uniqueness
+# from .check_finding_uniqueness import check_finding_uniqueness
 from census_historical_migration.invalid_record import InvalidRecord
 
 from .check_finding_award_references_pattern import award_references_pattern
@@ -113,7 +113,8 @@
     finding_reference_pattern,
     no_repeat_findings,
     findings_grid_validation,
-    check_finding_uniqueness,
+    # See ticket #4385 for more information on why this check is disabled
+    # check_finding_uniqueness,
 ]
 
 additional_eins_checks = general_checks + [
@@ -158,7 +159,8 @@
 require_gsa_migration_flag = [
     findings_grid_validation,
     finding_reference_pattern,
-    check_finding_uniqueness,
+    # See ticket #4385 for more information on why this check is disabled
+    # check_finding_uniqueness,
 ]
 
 

backend/audit/intakelib/constants.py

@@ -9,4 +9,5 @@
 FEDERAL_AGENCY_PREFIX = "federal_agency_prefix"
 THREE_DIGIT_EXTENSION = "three_digit_extension"
 SECTION_NAME = "section_name"
+VERSION = "version"
 XLSX_TEMPLATE_DEFINITION_DIR = settings.XLSX_TEMPLATE_JSON_DIR

backend/audit/intakelib/mapping_meta.py

@@ -1,5 +1,6 @@
 from .constants import (
     SECTION_NAME,
+    VERSION,
 )
 
 from .mapping_util import (
@@ -9,4 +10,5 @@
 
 meta_mapping: FieldMapping = {
     SECTION_NAME: (f"Meta.{SECTION_NAME}", _set_by_path),
+    VERSION: (f"Meta.{VERSION}", _set_by_path),
 }

backend/audit/intakelib/transforms/runners.py

@@ -1,6 +1,8 @@
 import logging
 from copy import deepcopy
 
+from .xform_clean_version_value import remove_equals_and_quotes
+
 from .xform_resize_award_references import resize_award_reference
 
 from .xform_all_amount_expended_need_to_be_integers import (
@@ -93,6 +95,7 @@ def run_all_secondary_auditors_transforms(ir):
 
 general_transforms = [
     convert_to_stripped_string,
+    remove_equals_and_quotes,
 ]
 
 notes_to_sefa_transforms = general_transforms + [

backend/audit/intakelib/transforms/xform_clean_version_value.py

@@ -0,0 +1,20 @@
+import logging
+from audit.intakelib.intermediate_representation import (
+    get_range_by_name,
+    replace_range_by_name,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def remove_equals_and_quotes(ir):
+    versions = get_range_by_name(ir, "version")
+    new_values = list(
+        map(
+            lambda v: v.replace("=", "").replace('"', "") if v else v,
+            versions["values"],
+        )
+    )
+    new_ir = replace_range_by_name(ir, "version", new_values)
+
+    return new_ir