DO NOT MERGE UNLESS WE NEED IT fix(s3): hotfix to revert back to old s3 paths

cypress/e2e/data-publication/CombinedMLAR.spec.js

@@ -19,13 +19,13 @@ onlyOn(!isBeta(HOST), () => {
         cy.visit(`${HOST}/data-publication/modified-lar/${year}`)
         cy.request({
           method: 'HEAD',
-          url: `${fileServerDomain}/modified-lar/combined-mlar/${year}/${year}_combined_mlar_header.zip`,
+          url: `${fileServerDomain}/prod/dynamic-data/combined-mlar/${year}/header/${year}_combined_mlar_header.zip`,
         })
           .its('status')
           .should('equal', 200)
         cy.request({
           method: 'HEAD',
-          url: `${fileServerDomain}/modified-lar/combined-mlar/${year}/${year}_combined_mlar.zip`,
+          url: `${fileServerDomain}/prod/dynamic-data/combined-mlar/${year}/${year}_combined_mlar.zip`,
         })
           .its('status')
           .should('equal', 200)

src/common/S3Integrations.jsx

@@ -1,5 +1,5 @@
-import { useEffect, useState } from 'react'
-import { Provider, useDispatch, useSelector } from 'react-redux'
+import React, { useEffect, useState } from 'react'
+import { useDispatch, useSelector, Provider } from 'react-redux'
 import ExternalLink from './ExternalLink'
 import LoadingIcon from './LoadingIcon'
 import { humanFileSize, isBigFile } from './numberServices'
@@ -27,39 +27,21 @@ export const useS3FileHeaders = (url, shouldFetch) => {
     setCurrHeaders(null)
     if (!shouldFetch) return
 
-    fetch(url, { method: 'HEAD' })
-      .then((response) => {
-        // catch errors errors served up from Akamai on 404s, return empty headers to hide file which
-        // mimics current prod behavior
-        // TODO: potentially remove after [GHE]/HMDA-Operations/hmda-devops/issues/5275 is resolved
-        if (!response.ok) {
-          dispatch(saveHeaders({ url, headers: {} }))
-          setCurrHeaders({})
-          return
-        }
-        const hdrs = ['last-modified', 'Content-Length']
-        const [lastMod, size] = hdrs.map((h) => response.headers.get(h))
-        let changeDate
+    fetch(url, { method: 'HEAD' }).then((response) => {
+      const hdrs = ['last-modified', 'Content-Length']
+      const [lastMod, size] = hdrs.map((h) => response.headers.get(h))
+      let changeDate
 
-        if (lastMod) {
-          const newDate = new Date(lastMod)
-          newDate.setHours(newDate.getHours() - 5) // Convert GMT to ET
-          changeDate = newDate.toDateString()
-        }
+      if (lastMod) {
+        const newDate = new Date(lastMod)
+        newDate.setHours(newDate.getHours() - 5) // Convert GMT to ET
+        changeDate = newDate.toDateString()
+      }
 
-        const headers = { changeDate, size }
-        dispatch(saveHeaders({ url, headers }))
-        setCurrHeaders(headers)
-      })
-      // catch CORS errors served up from Akamai on 404s, return empty headers to hide file which
-      // mimics current prod behavior
-      // TODO: potentially remove after [GHE]/HMDA-Operations/hmda-devops/issues/5275 is resolved
-      .catch((error) => {
-        if (error.name === 'TypeError' && error.message.includes('Failed to fetch')) {
-          dispatch(saveHeaders({ url, headers: {} }))
-          setCurrHeaders({})
-        }
-      })
+      const headers = { changeDate, size }
+      dispatch(saveHeaders({ url, headers }))
+      setCurrHeaders(headers)
+    })
   }, [url])
 
   if (!shouldFetch) return null
@@ -160,10 +142,9 @@ function LastUpdated({ url, isDocs }) {
   return (
     <div className={cname.join(' ')}>
       <S3LargeFileWarning show={isBigFile(readableSize)} />
-      {/* {TODO: Reenable size after [GHE]/HMDA-Operations/hmda-devops/issues/5275} */}
-      {/* <div>
+      <div>
         <span className='label'>Size:</span> {readableSize}
-      </div> */}
+      </div>
       <div>
         <span className='label'>Updated:</span> {headers.changeDate}
       </div>

src/common/constants/dev-beta-config.json

@@ -20,7 +20,7 @@
       }
     }
   ],
-  "fileServerDomain": "https://files.ffiec.cfpb.gov",
+  "fileServerDomain": "https://s3.amazonaws.com/cfpb-hmda-public",
   "publicationReleaseYear": "2023", 
   "mlarReleaseYear": "2023",
   "maintenanceMode": false,

src/common/constants/dev-config.json

@@ -20,7 +20,7 @@
       }
     }
   ],
-  "fileServerDomain": "https://files.ffiec.cfpb.gov",
+  "fileServerDomain": "https://s3.amazonaws.com/cfpb-hmda-public",
   "publicationReleaseYear": "2024", 
   "mlarReleaseYear": "2024",
   "maintenanceMode": false,

src/common/constants/prod-beta-config.json

@@ -1,7 +1,7 @@
 {
   "name": "prod-beta",
   "announcement": [],
-  "fileServerDomain": "https://files.ffiec.cfpb.gov",
+  "fileServerDomain": "https://s3.amazonaws.com/cfpb-hmda-public",
   "publicationReleaseYear": "2020",
   "maintenanceMode": false,
   "filingAnnouncement": null,

src/common/constants/prod-config.json

@@ -20,7 +20,7 @@
       }
     }
   ],
-  "fileServerDomain": "https://files.ffiec.cfpb.gov",
+  "fileServerDomain": "https://s3.amazonaws.com/cfpb-hmda-public",
   "publicationReleaseYear": "2024",
   "mlarReleaseYear": "2024",
   "maintenanceMode": false,

src/data-browser/maps/filterUtils.jsx

@@ -4,7 +4,7 @@ function getFile(y, g, v, val) {
   const { fileServerDomain } = getDefaultConfig(window.location.hostname)
 
   const dec = decodeURIComponent(val)
-  return `${fileServerDomain}/data-browser/maps/${y}/${g}-${v}-${dec
+  return `${fileServerDomain}/prod/data-browser/filter-data/${y}/${g}-${v}-${dec
     .replace(/[^a-z0-9]/gi, '-')
     .toLowerCase()}.json`
 }

src/data-publication/constants/dynamic-dataset.js

@@ -4,36 +4,36 @@ const { fileServerDomain } = getDefaultConfig(window.location.hostname)
 
 export const DYNAMIC_DATASET = {
   2024: {
-    lar: `${fileServerDomain}/dynamic-data/2024/2024_lar.zip`,
-    ts: `${fileServerDomain}/dynamic-data/2024/2024_ts.zip`,
+    lar: `${fileServerDomain}/prod/dynamic-data/2024/2024_lar.zip`,
+    ts: `${fileServerDomain}/prod/dynamic-data/2024/2024_ts.zip`,
   },
   2023: {
-    lar: `${fileServerDomain}/dynamic-data/2023/2023_lar.zip`,
-    ts: `${fileServerDomain}/dynamic-data/2023/2023_ts.zip`,
+    lar: `${fileServerDomain}/prod/dynamic-data/2023/2023_lar.zip`,
+    ts: `${fileServerDomain}/prod/dynamic-data/2023/2023_ts.zip`,
   },
   2022: {
-    lar: `${fileServerDomain}/dynamic-data/2022/2022_lar.zip`,
-    ts: `${fileServerDomain}/dynamic-data/2022/2022_ts.zip`,
+    lar: `${fileServerDomain}/prod/dynamic-data/2022/2022_lar.zip`,
+    ts: `${fileServerDomain}/prod/dynamic-data/2022/2022_ts.zip`,
   },
   2021: {
-    lar: `${fileServerDomain}/dynamic-data/2021/2021_lar.zip`,
-    ts: `${fileServerDomain}/dynamic-data/2021/2021_ts.zip`,
+    lar: `${fileServerDomain}/prod/dynamic-data/2021/2021_lar.zip`,
+    ts: `${fileServerDomain}/prod/dynamic-data/2021/2021_ts.zip`,
   },
   2020: {
-    lar: `${fileServerDomain}/dynamic-data/2020/2020_lar.zip`,
-    ts: `${fileServerDomain}/dynamic-data/2020/2020_ts.zip`,
+    lar: `${fileServerDomain}/prod/dynamic-data/2020/2020_lar.zip`,
+    ts: `${fileServerDomain}/prod/dynamic-data/2020/2020_ts.zip`,
   },
   2019: {
-    lar: `${fileServerDomain}/dynamic-data/2019/2019_lar.zip`,
-    ts: `${fileServerDomain}/dynamic-data/2019/2019_ts.zip`,
+    lar: `${fileServerDomain}/prod/dynamic-data/2019/2019_lar.zip`,
+    ts: `${fileServerDomain}/prod/dynamic-data/2019/2019_ts.zip`,
   },
   2018: {
-    lar: `${fileServerDomain}/dynamic-data/2018/2018_lar.zip`,
-    ts: `${fileServerDomain}/dynamic-data/2018/2018_ts.zip`,
+    lar: `${fileServerDomain}/prod/dynamic-data/2018/2018_lar.zip`,
+    ts: `${fileServerDomain}/prod/dynamic-data/2018/2018_ts.zip`,
   },
   2017: {
-    lar: `${fileServerDomain}/dynamic-data/2017/dynamic_lar_ultimate_2017.txt`,
-    ts: `${fileServerDomain}/dynamic-data/2017/dynamic_ts_ultimate_2017.txt`,
+    lar: `${fileServerDomain}/prod/dynamic-data/dynamic_lar_ultimate_2017.txt`,
+    ts: `${fileServerDomain}/prod/dynamic-data/dynamic_ts_ultimate_2017.txt`,
     lar_spec:
       'https://github.com/cfpb/hmda-platform/blob/v1.x/Documents/2017_Dynamic_LAR_Spec.csv',
     ts_spec:

src/data-publication/constants/one-year-datasets.jsx

@@ -14,14 +14,14 @@ export const ONE_YEAR_DATASET = {
       {
         dataKey: 'lar',
         label: 'Loan/Application Records (LAR)',
-        csv: `${fileServerDomain}/static-data/one-year/2023/2023_public_lar_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2023/2023_public_lar_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2023/2023_public_lar_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2023/2023_public_lar_one_year_pipe.zip`,
       },
       {
         dataKey: 'ts',
         label: 'Transmittal Sheet Records (TS)',
-        csv: `${fileServerDomain}/static-data/one-year/2023/2023_public_ts_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2023/2023_public_ts_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2023/2023_public_ts_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2023/2023_public_ts_one_year_pipe.zip`,
       },
       {
         dataKey: 'panel',
@@ -31,8 +31,8 @@ export const ONE_YEAR_DATASET = {
       {
         dataKey: 'msamd',
         label: 'MSA/MD Description',
-        csv: `${fileServerDomain}/static-data/one-year/2023/2023_public_msamd_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2023/2023_public_msamd_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2023/2023_public_msamd_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2023/2023_public_msamd_one_year_pipe.zip`,
       },
     ],
   },
@@ -42,26 +42,26 @@ export const ONE_YEAR_DATASET = {
       {
         dataKey: 'lar',
         label: 'Loan/Application Records (LAR)',
-        csv: `${fileServerDomain}/static-data/one-year/2022/2022_public_lar_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2022/2022_public_lar_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2022/2022_public_lar_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2022/2022_public_lar_one_year_pipe.zip`,
       },
       {
         dataKey: 'ts',
         label: 'Transmittal Sheet Records (TS)',
-        csv: `${fileServerDomain}/static-data/one-year/2022/2022_public_ts_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2022/2022_public_ts_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2022/2022_public_ts_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2022/2022_public_ts_one_year_pipe.zip`,
       },
       {
         dataKey: 'panel',
         label: 'Reporter Panel',
-        csv: `${fileServerDomain}/static-data/one-year/2022/2022_public_panel_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2022/2022_public_panel_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2022/2022_public_panel_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2022/2022_public_panel_one_year_pipe.zip`,
       },
       {
         dataKey: 'msamd',
         label: 'MSA/MD Description',
-        csv: `${fileServerDomain}/static-data/one-year/2022/2022_public_msamd_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2022/2022_public_msamd_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2022/2022_public_msamd_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2022/2022_public_msamd_one_year_pipe.zip`,
       },
     ],
   },
@@ -71,26 +71,26 @@ export const ONE_YEAR_DATASET = {
       {
         dataKey: 'lar',
         label: 'Loan/Application Records (LAR)',
-        csv: `${fileServerDomain}/static-data/one-year/2021/2021_public_lar_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2021/2021_public_lar_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2021/2021_public_lar_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2021/2021_public_lar_one_year_pipe.zip`,
       },
       {
         dataKey: 'ts',
         label: 'Transmittal Sheet Records (TS)',
-        csv: `${fileServerDomain}/static-data/one-year/2021/2021_public_ts_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2021/2021_public_ts_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2021/2021_public_ts_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2021/2021_public_ts_one_year_pipe.zip`,
       },
       {
         dataKey: 'panel',
         label: 'Reporter Panel',
-        csv: `${fileServerDomain}/static-data/one-year/2021/2021_public_panel_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2021/2021_public_panel_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2021/2021_public_panel_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2021/2021_public_panel_one_year_pipe.zip`,
       },
       {
         dataKey: 'msamd',
         label: 'MSA/MD Description',
-        csv: `${fileServerDomain}/static-data/one-year/2021/2021_public_msamd_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2021/2021_public_msamd_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2021/2021_public_msamd_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2021/2021_public_msamd_one_year_pipe.zip`,
       },
     ],
   },
@@ -100,26 +100,26 @@ export const ONE_YEAR_DATASET = {
       {
         dataKey: 'lar',
         label: 'Loan/Application Records (LAR)',
-        csv: `${fileServerDomain}/static-data/one-year/2020/2020_public_lar_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2020/2020_public_lar_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2020/2020_public_lar_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2020/2020_public_lar_one_year_pipe.zip`,
       },
       {
         dataKey: 'ts',
         label: 'Transmittal Sheet Records (TS)',
-        csv: `${fileServerDomain}/static-data/one-year/2020/2020_public_ts_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2020/2020_public_ts_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2020/2020_public_ts_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2020/2020_public_ts_one_year_pipe.zip`,
       },
       {
         dataKey: 'panel',
         label: 'Reporter Panel',
-        csv: `${fileServerDomain}/static-data/one-year/2020/2020_public_panel_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2020/2020_public_panel_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2020/2020_public_panel_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2020/2020_public_panel_one_year_pipe.zip`,
       },
       {
         dataKey: 'msamd',
         label: 'MSA/MD Description',
-        csv: `${fileServerDomain}/static-data/one-year/2020/2020_public_msamd_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2020/2020_public_msamd_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2020/2020_public_msamd_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2020/2020_public_msamd_one_year_pipe.zip`,
       },
     ],
   },
@@ -136,26 +136,26 @@ export const ONE_YEAR_DATASET = {
       {
         dataKey: 'lar',
         label: 'Loan/Application Records (LAR)',
-        csv: `${fileServerDomain}/static-data/one-year/2019/2019_public_lar_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2019/2019_public_lar_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2019/2019_public_lar_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2019/2019_public_lar_one_year_pipe.zip`,
       },
       {
         dataKey: 'ts',
         label: 'Transmittal Sheet Records (TS)',
-        csv: `${fileServerDomain}/static-data/one-year/2019/2019_public_ts_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2019/2019_public_ts_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2019/2019_public_ts_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2019/2019_public_ts_one_year_pipe.zip`,
       },
       {
         dataKey: 'panel',
         label: 'Reporter Panel',
-        csv: `${fileServerDomain}/static-data/one-year/2019/2019_public_panel_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2019/2019_public_panel_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2019/2019_public_panel_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2019/2019_public_panel_one_year_pipe.zip`,
       },
       {
         dataKey: 'msamd',
         label: 'MSA/MD Description',
-        csv: `${fileServerDomain}/static-data/one-year/2019/2019_public_msamd_one_year_csv.zip`,
-        txt: `${fileServerDomain}/static-data/one-year/2019/2019_public_msamd_one_year_pipe.zip`,
+        csv: `${fileServerDomain}/prod/one-year-data/2019/2019_public_msamd_one_year_csv.zip`,
+        txt: `${fileServerDomain}/prod/one-year-data/2019/2019_public_msamd_one_year_pipe.zip`,
       },
     ],
   },