Skip to content

Commit

Permalink
Merge pull request #4899 from kgudel/allow2025
Browse files Browse the repository at this point in the history
Set up 2025 in the HMDA-Platform
  • Loading branch information
PatrickGoRaft authored Sep 17, 2024
2 parents ecb4fc3 + 6f07fb4 commit 68131c0
Show file tree
Hide file tree
Showing 11 changed files with 803 additions and 6 deletions.
315 changes: 315 additions & 0 deletions common/src/main/resources/2025EditsDescriptions.txt

Large diffs are not rendered by default.

315 changes: 315 additions & 0 deletions common/src/main/resources/2025QuarterlyEditsDescriptions.txt

Large diffs are not rendered by default.

26 changes: 24 additions & 2 deletions common/src/main/resources/reference.conf
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ hmda {

rules {
yearly-filing {
years-allowed = "2018,2019,2020,2021,2022,2023,2024"
years-allowed = "2018,2019,2020,2021,2022,2023,2024,2025"
years-allowed = ${?RULES_YEARLY_FILING_YEARS_ALLOWED}
}

Expand Down Expand Up @@ -179,6 +179,20 @@ hmda {
edits.descriptions.filename = ${?EDIT_DESCRIPTIONS_FILENAME_2024}
year = 2024
}
2024Quarter {
ts.length = 15
lar.length = 110
edits.descriptions.filename = "2025QuarterlyEditsDescriptions.txt"
edits.descriptions.filename = ${?EDIT_DESCRIPTIONS_FILENAME_2025_Q}
year = 2024
}
2024 {
ts.length = 15
lar.length = 110
edits.descriptions.filename = "2025EditsDescriptions.txt"
edits.descriptions.filename = ${?EDIT_DESCRIPTIONS_FILENAME_2025}
year = 2024
}
}

census {
Expand Down Expand Up @@ -209,7 +223,11 @@ hmda {
}
2024 {
filename = "ffiec_census_2024.txt"
filename = ${?2023_CENSUS_FILENAME}
filename = ${?2024_CENSUS_FILENAME}
}
2025 {
filename = "ffiec_census_2024.txt"
filename = ${?2024_CENSUS_FILENAME}
}
}
}
Expand Down Expand Up @@ -243,6 +261,10 @@ hmda {
fields.filename = "FullCountyLoanLimitList2024.txt"
fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME_2024}
}
2025 {
fields.filename = "FullCountyLoanLimitList2024.txt"
fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME_2024}
}
}

kafka {
Expand Down
13 changes: 12 additions & 1 deletion common/src/main/scala/hmda/census/records/CensusRecords.scala
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ object CensusRecords {
private val censusFileName2022 = config.getString("hmda.census.fields.2022.filename")
private val censusFileName2023 = config.getString("hmda.census.fields.2023.filename")
private val censusFileName2024 = config.getString("hmda.census.fields.2024.filename")
private val censusFileName2025 = config.getString("hmda.census.fields.2025.filename")


val (
Expand Down Expand Up @@ -81,11 +82,17 @@ object CensusRecords {
indexedSmallCounty2023: Map[String, Census]
) = getCensus(censusFileName2023)

val (
val (
indexedTract2024: Map[String, Census],
indexedCounty2024: Map[String, Census],
indexedSmallCounty2024: Map[String, Census]
) = getCensus(censusFileName2024)

val (
indexedTract2025: Map[String, Census],
indexedCounty2025: Map[String, Census],
indexedSmallCounty2025: Map[String, Census]
) = getCensus(censusFileName2025)

def yearTractMap(year: Int): Map[String, Census] = {
year match {
Expand All @@ -103,6 +110,8 @@ object CensusRecords {
indexedTract2023
case 2024 =>
indexedTract2024
case 2025 =>
indexedTract2025
case _ =>
indexedTract2024
}
Expand All @@ -124,6 +133,8 @@ object CensusRecords {
indexedCounty2023
case 2024 =>
indexedCounty2024
case 2025 =>
indexedCounty2025
case _ =>
indexedCounty2024
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,10 @@ object EditDescriptionLookup {
config.getString("hmda.filing.2024Quarter.edits.descriptions.filename")
val editDescriptionFileName2024 =
config.getString("hmda.filing.2024.edits.descriptions.filename")
val editDescriptionFileName2025Quarter =
config.getString("hmda.filing.2025Quarter.edits.descriptions.filename")
val editDescriptionFileName2025 =
config.getString("hmda.filing.2025.edits.descriptions.filename")
def editDescriptionList(file: Iterable[String]): Iterable[EditDescription] =
file
.drop(1)
Expand All @@ -59,6 +63,8 @@ object EditDescriptionLookup {
val editDescriptionLines2023 = fileLines(s"/$editDescriptionFileName2023")
val editDescriptionLines2024Quarter = fileLines(s"/$editDescriptionFileName2024Quarter")
val editDescriptionLines2024 = fileLines(s"/$editDescriptionFileName2024")
val editDescriptionLines2025Quarter = fileLines(s"/$editDescriptionFileName2025Quarter")
val editDescriptionLines2025 = fileLines(s"/$editDescriptionFileName2025")


val editDescriptionMap2018 = editDescriptionMap(editDescriptionLines2018)
Expand All @@ -73,6 +79,8 @@ object EditDescriptionLookup {
val editDescriptionMap2023 = editDescriptionMap(editDescriptionLines2023)
val editDescriptionMap2024Quarter = editDescriptionMap(editDescriptionLines2024Quarter)
val editDescriptionMap2024 = editDescriptionMap(editDescriptionLines2024)
val editDescriptionMap2025Quarter = editDescriptionMap(editDescriptionLines2025Quarter)
val editDescriptionMap2025 = editDescriptionMap(editDescriptionLines2025)


def mapForPeriod(period: Period): Map[String, EditDescription] =
Expand All @@ -89,6 +97,8 @@ object EditDescriptionLookup {
case Period(2023, None) => editDescriptionMap2023
case Period(2024, Some(_)) => editDescriptionMap2024Quarter
case Period(2024, None) => editDescriptionMap2024
case Period(2025, Some(_)) => editDescriptionMap2025Quarter
case Period(2025, None) => editDescriptionMap2025
case _ => editDescriptionMap2021
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,9 @@ private class HmdaFileValidationHttpApi(implicit mat: Materializer) {

case "validity" =>
checkValidity(ts, ts.LEI, ctx, TsValidationError)

case "quality" =>
checkQuality(ts, ts.LEI, ctx)
}

val maybeErrors = validation.leftMap(xs => xs.toList).toEither
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,13 @@ import hmda.model.filing.ts.TransmittalSheet
import hmda.validation.context.ValidationContext
import hmda.validation.rules.ts.syntactical.{ S300, S303, S302 }
import hmda.validation.rules.ts.validity._
import hmda.validation.rules.ts.quality._
// $COVERAGE-OFF$
private[engine] object TsEngine2025 extends ValidationEngine[TransmittalSheet] {

override def syntacticalChecks(ctx: ValidationContext) = Vector(
S300,
S302.withContext(ctx),
S303.withContext(ctx)
S302.withContext(ctx)
)

override def validityChecks(ctx: ValidationContext) = Vector(
Expand All @@ -27,5 +27,9 @@ private[engine] object TsEngine2025 extends ValidationEngine[TransmittalSheet] {
V719_2
)

override def qualityChecks(ctx: ValidationContext) = Vector(
Q303.withContext(ctx)
)

}
// $COVERAGE-ON$
8 changes: 7 additions & 1 deletion hmda/src/main/scala/hmda/validation/engine/package.scala
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ package object engine {
case (2023, None) => TsEngine2023
case (2024, Some(_)) => TsEngine2024Q
case (2024, None) => TsEngine2024
case (2025, Some(_)) => TsEngine2025Q
case (2025, None) => TsEngine2025

case _ => TsEngine2022 // TODO: determine what engine to pick if the user enters a year that is not covered
}
Expand All @@ -36,6 +38,8 @@ package object engine {
case (2023, None) => TsLarEngine2023
case (2024, Some(_)) => TsLarEngine2024Q
case (2024, None) => TsLarEngine2024
case (2025, Some(_)) => TsLarEngine2025Q
case (2025, None) => TsLarEngine2025
case _ =>
TsLarEngine2022 // TODO: determine what engine to pick if the user enters a year that is not covered
}
Expand All @@ -53,7 +57,9 @@ package object engine {
case (2023, Some(_)) => LarEngine2023Q
case (2023, None) => LarEngine2023
case (2024, Some(_)) => LarEngine2024Q
case (2024, None) => LarEngine2024
case (2024, None) => LarEngine2024
case (2025, Some(_)) => LarEngine2025Q
case (2025, None) => LarEngine2025
case _ =>
LarEngine2022 // TODO: determine what engine to pick if the user enters a year that is not covered
}
Expand Down
13 changes: 13 additions & 0 deletions hmda/src/main/scala/hmda/validation/filing/ValidationFlow.scala
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ object ValidationFlow extends ColumnDataFormatter {
validationEngine.checkSyntactical(ts, ts.LEI, validationContext, TsValidationError)
case "validity" =>
validationEngine.checkValidity(ts, ts.LEI, validationContext, TsValidationError)
case "quality" =>
validationEngine.checkQuality(ts, ts.LEI, validationContext)
}
(ts, errors)
}
Expand Down Expand Up @@ -153,6 +155,17 @@ object ValidationFlow extends ColumnDataFormatter {
)
): _*
)
case "Q303" =>
ListMap(
affectedFields.map(field =>
(
field,
"Provided: " + ts.valueOf(field) + ", Expected: " + institution
.getOrElse(Institution.empty)
.valueOf(field)
)
): _*
)
case "V718" =>
val quarter =
period.quarter match {
Expand Down
24 changes: 24 additions & 0 deletions hmda/src/main/scala/hmda/validation/rules/ts/quality/Q303.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package hmda.validation.rules.ts.quality

import hmda.model.filing.ts.TransmittalSheet
import hmda.model.institution.Institution
import hmda.validation.context.ValidationContext
import hmda.validation.dsl.ValidationResult
import hmda.validation.rules.{ EditCheck, IfInstitutionPresentIn }
import hmda.validation.dsl.PredicateCommon._
import hmda.validation.dsl.PredicateSyntax._

object Q303 {
def withContext(ctx: ValidationContext): EditCheck[TransmittalSheet] =
IfInstitutionPresentIn(ctx) { new Q303(_) }

}

class Q303 private (institution: Institution) extends EditCheck[TransmittalSheet] {
override def name: String = "Q303"

override def apply(ts: TransmittalSheet): ValidationResult =
(ts.LEI.toLowerCase is equalTo(institution.LEI.toLowerCase)) and
(ts.agency.code is equalTo(institution.agency.code)) and
(ts.taxId is equalTo(institution.taxId.getOrElse("")))
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
package hmda.validation.rules.ts.quality

import hmda.model.filing.ts.TransmittalSheet
import hmda.model.institution._
import hmda.validation.rules.EditCheck
import hmda.validation.rules.ts.TsEditCheckSpec
import hmda.model.filing.ts.TsGenerators._
import hmda.validation.context.ValidationContext

class Q303Spec extends TsEditCheckSpec {

private var institution: Institution = _

override def check: EditCheck[TransmittalSheet] =
Q303.withContext(ValidationContext(Some(institution)))

property("Pass when LEI, Agency and Tax ID are reported correctly") {
forAll(tsGen) { ts =>
whenInstitution(ts.LEI, ts.agency, ts.taxId)
ts.mustPass
}
}

property("Pass when LEI is a different case") {
forAll(tsGen) { ts =>
whenInstitution(ts.LEI.toLowerCase, ts.agency, ts.taxId)
ts.mustPass
val upperCaseTs = ts.copy(LEI = ts.LEI.toUpperCase)
upperCaseTs.mustPass
}
}

property("Fail when LEI is reported incorrectly") {
forAll(tsGen) { ts =>
whenInstitution(ts.LEI + "x", ts.agency, ts.taxId)
ts.mustFail
}
}

property("Fail when Agency is reported incorrectly") {
forAll(tsGen) { ts =>
whenever(ts.agency != CFPB) {
whenInstitution(ts.LEI, CFPB, ts.taxId)
ts.mustFail
}
}
}

property("Fail when Tax ID is reported incorrectly") {
forAll(tsGen) { ts =>
whenInstitution(ts.LEI, ts.agency, ts.taxId + "x")
ts.mustFail
}
}

property("Fail when LEI, Agency and Tax ID are reported incorrectly") {
forAll(tsGen) { ts =>
whenever(ts.agency != CFPB) {
whenInstitution(ts.LEI + "x", CFPB, ts.taxId + "x")
ts.mustFail
}
}
}
private def whenInstitution(lei: String,
agency: Agency,
taxId: String): Unit = {
institution = Institution.empty.copy(
LEI = lei,
agency = agency,
taxId = Some(taxId)
)
}

}

0 comments on commit 68131c0

Please sign in to comment.