diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 26c68eee69..b296f84a1a 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -14,22 +14,82 @@ jobs: uses: actions/checkout@v4 - name: Setup JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: - java-version: '17' + java-version: '11' distribution: 'adopt' - - name: Run Sbt Tests + - name: Run hmda-platform Tests run: | - touch log-file - sbt test > log-file + sbt "project hmda-platform" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run check-digit Tests + run: | + sbt "project check-digit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run common Tests + run: | + sbt "project common" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run data-browser Tests + run: | + sbt "project data-browser" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run hmda-analytics Tests + run: | + sbt "project hmda-analytics" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run hmda-data-publisher Tests + run: | + sbt "project hmda-data-publisher" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run hmda-quarterly-data-service Tests + run: | + sbt "project hmda-quarterly-data-service" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run hmda-reporting Tests + run: | + sbt "project hmda-reporting" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run institutions-api Tests + run: | + sbt "project institutions-api" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run modified-lar Tests + run: | + sbt "project modified-lar" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run rate-limit Tests + run: | + sbt "project rate-limit" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run ratespread-calculator Tests + run: | + sbt "project ratespread-calculator" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file + continue-on-error: true + + - name: Run submission-errors Tests + run: | + sbt "project submission-errors" "testOnly -- -l actions-ignore" 2>&1 | tee -a log-file continue-on-error: true - name: Check Test Results run: | - if [ $(cat log-file | grep -E "TEST FAILED|TESTS FAILED|Failed tests" | wc -l) -gt 0 ]; then - echo "Unit tests failed." + if [ $(grep 'All tests passed.' log-file | wc -l) -ne 13 ]; then + echo "One or more projects had failures. Please review the logs." exit 1 else - echo "Unit tests passed." + echo "All tests passed." + exit 0 fi diff --git a/build.sbt b/build.sbt index d73b8b36cd..009fb664bd 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ import BuildSettings._ import sbtassembly.AssemblyPlugin.autoImport.assemblyMergeStrategy import com.typesafe.sbt.packager.docker._ -lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, embeddedPg, embeddedPgSupport, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi) +lazy val commonDeps = Seq(logback, scalaTest, scalaCheck, akkaHttpSprayJson, testContainers, apacheCommonsIO, s3Mock,log4jToSlf4j, kubernetesApi) lazy val sparkDeps = Seq( diff --git a/common/src/main/resources/reference.conf b/common/src/main/resources/reference.conf index ebbedb3709..5a55706ddb 100644 --- a/common/src/main/resources/reference.conf +++ b/common/src/main/resources/reference.conf @@ -314,7 +314,7 @@ db { url = "jdbc:postgresql://"${dbconfig.host}":"${dbconfig.port}"/"${dbconfig.database}"?user="${dbconfig.user}"&password="${dbconfig.password}"&sslmode="${dbconfig.ssl}"&"${dbconfig.additionalConfig} url = ${?JDBC_URL} numThreads = 4 - queueSize = 10 + queueSize = 100 connectionTimeout = 20000 validationTimeout = 10000 } diff --git a/common/src/main/scala/hmda/util/CSVConsolidator.scala b/common/src/main/scala/hmda/util/CSVConsolidator.scala index 327e9969b0..308ef2e107 100644 --- a/common/src/main/scala/hmda/util/CSVConsolidator.scala +++ b/common/src/main/scala/hmda/util/CSVConsolidator.scala @@ -10,11 +10,11 @@ object CSVConsolidator { seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.mkString(",") } def listDeDupeToList(seqToDeDupe: Seq[String]) = { - seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.toList + seqToDeDupe.mkString(",").toLowerCase().trim.split("\\s*,\\s*").distinct.filter(! _.isEmpty).toList } def stringDeDupeToList(stringToDeDupe: String) = { - stringToDeDupe.toLowerCase().trim.split("\\s*,\\s*").distinct.toList + stringToDeDupe.toLowerCase().trim.split("\\s*,\\s*").distinct.filter(! _.isEmpty).toList } def stringDeDupeToString(stringToDeDupe: String) = { diff --git a/common/src/test/scala/hmda/utils/EmbeddedPostgres.scala b/common/src/test/scala/hmda/utils/EmbeddedPostgres.scala index cc25c0eb76..4cbfc94b4a 100644 --- a/common/src/test/scala/hmda/utils/EmbeddedPostgres.scala +++ b/common/src/test/scala/hmda/utils/EmbeddedPostgres.scala @@ -1,7 +1,7 @@ package hmda.utils import org.scalatest.{ BeforeAndAfterAll, BeforeAndAfterEach, Suite } -import ru.yandex.qatools.embed.postgresql.distribution.Version +import org.testcontainers.containers.FixedHostPortGenericContainer import slick.basic.DatabaseConfig import slick.dbio.DBIO import slick.jdbc.JdbcProfile @@ -16,7 +16,7 @@ import scala.concurrent.ExecutionContext.Implicits.global * after each test and finally deletes all tables before the suite is complete and tears down the database */ trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self: Suite => - private val embeddedPg = new ru.yandex.qatools.embed.postgresql.EmbeddedPostgres(Version.V11_1) + private val testContainer = new FixedHostPortGenericContainer("postgres:12") val dbHoconpath = "embedded-pg" val dbConfig = DatabaseConfig.forConfig[JdbcProfile](dbHoconpath) @@ -48,7 +48,11 @@ trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self: } override protected def beforeAll(): Unit = { - embeddedPg.start("localhost", 5432, "postgres", "postgres", "postgres") + testContainer.withEnv("POSTGRES_USER", "postgres") + testContainer.withEnv("POSTGRES_PASSWORD", "postgres") + testContainer.withEnv("POSTGRES_DB", "postgres") + testContainer.withFixedExposedPort(5432, 5432) + testContainer.start() executeSQL(removeAllTables) loadSqlFileFromResources(bootstrapSqlFile) super.beforeAll() @@ -56,7 +60,7 @@ trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self: override protected def afterAll(): Unit = { executeSQL(removeAllTables) - embeddedPg.close() + testContainer.stop() super.afterAll() } diff --git a/data-browser/src/test/resources/application.conf b/data-browser/src/test/resources/application.conf index 11f7669ba2..04c7c8571f 100644 --- a/data-browser/src/test/resources/application.conf +++ b/data-browser/src/test/resources/application.conf @@ -17,7 +17,7 @@ embedded-pg { profile = slick.jdbc.PostgresProfile url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres" numThreads = 2 - queueSize = 10 + queueSize = 100 connectionTimeout = 20000 validationTimeout = 10000 } diff --git a/data-browser/src/test/scala/hmda/dataBrowser/api/DataBrowserDirectivesSpec.scala b/data-browser/src/test/scala/hmda/dataBrowser/api/DataBrowserDirectivesSpec.scala index 4a3d163077..775afc1851 100644 --- a/data-browser/src/test/scala/hmda/dataBrowser/api/DataBrowserDirectivesSpec.scala +++ b/data-browser/src/test/scala/hmda/dataBrowser/api/DataBrowserDirectivesSpec.scala @@ -15,6 +15,9 @@ import monix.execution.schedulers.TestScheduler import org.scalamock.scalatest.MockFactory import org.scalatest.concurrent.ScalaFutures import org.scalatest.{ Assertion, Matchers, WordSpec } +import org.scalatest.Tag + +object CustomTag extends Tag("actions-ignore") class DataBrowserDirectivesSpec extends WordSpec with ScalatestRouteTest with Matchers with MockFactory with ScalaFutures { "DataBrowserDirectives" must { @@ -177,7 +180,7 @@ class DataBrowserDirectivesSpec extends WordSpec with ScalatestRouteTest with Ma } } - "extractYearsAndMsaAndStateAndCountyAndLEIBrowserFields prevents you from providing too many parameters" in { + "extractYearsAndMsaAndStateAndCountyAndLEIBrowserFields prevents you from providing too many parameters" taggedAs CustomTag in { val route: Route = failingRoute(extractMsaAndStateAndCountyAndInstitutionIdentifierBrowserFields) Get("/?msamds=34980&leis=BANK0&states=CA,AK&actions_taken=1,2,3&counties=19125&years=2018") ~> route ~> check { diff --git a/hmda-analytics/src/test/resources/application.conf b/hmda-analytics/src/test/resources/application.conf index 7b1e1511ec..3db78717c2 100644 --- a/hmda-analytics/src/test/resources/application.conf +++ b/hmda-analytics/src/test/resources/application.conf @@ -22,7 +22,7 @@ embedded-pg { profile = slick.jdbc.PostgresProfile url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres" numThreads = 2 - queueSize = 10 + queueSize = 100 connectionTimeout = 20000 validationTimeout = 10000 } diff --git a/hmda-dashboard/src/main/resources/application.conf b/hmda-dashboard/src/main/resources/application.conf index bf3639aeff..7372aedcf4 100644 --- a/hmda-dashboard/src/main/resources/application.conf +++ b/hmda-dashboard/src/main/resources/application.conf @@ -45,7 +45,7 @@ dashboard_db { url = "jdbc:postgresql://"${dbconfig.host}":"${dbconfig.port}"/"${dbconfig.database}"?user="${dbconfig.user}"&password="${dbconfig.password}"&sslmode="${dbconfig.ssl}"&"${dbconfig.additionalConfig} url = ${?JDBC_URL} numThreads = 10 - queueSize = 10 + queueSize = 100 connectionTimeout = 2000000 validationTimeout = 1000000 } diff --git a/hmda-data-publisher/src/test/resources/application.conf b/hmda-data-publisher/src/test/resources/application.conf index a5f74e24db..b40bfabf09 100644 --- a/hmda-data-publisher/src/test/resources/application.conf +++ b/hmda-data-publisher/src/test/resources/application.conf @@ -143,7 +143,7 @@ embedded-pg { profile = slick.jdbc.PostgresProfile url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres" numThreads = 2 - queueSize = 10 + queueSize = 100 connectionTimeout = 20000 validationTimeout = 10000 } diff --git a/hmda-reporting/src/test/resources/application.conf b/hmda-reporting/src/test/resources/application.conf index 3a814c4a1f..cb7156af9b 100644 --- a/hmda-reporting/src/test/resources/application.conf +++ b/hmda-reporting/src/test/resources/application.conf @@ -24,7 +24,7 @@ embedded-pg { profile = slick.jdbc.PostgresProfile url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres" numThreads = 2 - queueSize = 10 + queueSize = 100 connectionTimeout = 20000 validationTimeout = 10000 } diff --git a/hmda/src/test/scala/hmda/persistence/institution/InstitutionPersistenceSpec.scala b/hmda/src/test/scala/hmda/persistence/institution/InstitutionPersistenceSpec.scala index 965b5e0b6a..fc096e7ff2 100644 --- a/hmda/src/test/scala/hmda/persistence/institution/InstitutionPersistenceSpec.scala +++ b/hmda/src/test/scala/hmda/persistence/institution/InstitutionPersistenceSpec.scala @@ -12,6 +12,9 @@ import hmda.persistence.AkkaCassandraPersistenceSpec import akka.actor.typed.scaladsl.adapter._ import hmda.messages.institution.InstitutionCommands._ import hmda.messages.institution.InstitutionEvents._ +import org.scalatest.Tag + +object CustomTag extends Tag("actions-ignore") class InstitutionPersistenceSpec extends AkkaCassandraPersistenceSpec { @@ -44,7 +47,7 @@ class InstitutionPersistenceSpec extends AkkaCassandraPersistenceSpec { maybeInstitutionProbe.expectMessage(Some(sampleInstitution)) } - "not be created if it already exists" in { + "not be created if it already exists" taggedAs CustomTag in { val institutionPersistence = system.spawn(InstitutionPersistence.behavior("ABC12345"), actorName) institutionPersistence ! CreateInstitution(sampleInstitution, institutionProbe.ref) diff --git a/modified-lar/src/test/resources/application.conf b/modified-lar/src/test/resources/application.conf index fec3830241..051d447159 100644 --- a/modified-lar/src/test/resources/application.conf +++ b/modified-lar/src/test/resources/application.conf @@ -76,7 +76,7 @@ embedded-pg { profile = slick.jdbc.PostgresProfile url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres" numThreads = 2 - queueSize = 10 + queueSize = 100 connectionTimeout = 20000 validationTimeout = 10000 } diff --git a/project/BuildSettings.scala b/project/BuildSettings.scala index 5001efceeb..0da9def3cd 100644 --- a/project/BuildSettings.scala +++ b/project/BuildSettings.scala @@ -21,7 +21,7 @@ object BuildSettings { scalaVersion := buildScalaVersion, scalacOptions ++= Seq("-Xlint", "-deprecation", "-unchecked", "-feature", "-Ymacro-annotations"), assembly / aggregate := false, - Test / parallelExecution:= true, + Test / parallelExecution:= false, Test / fork := true, resolvers += "Akka library repository".at("https://repo.akka.io/maven"), commands += dockerPublishLocalSkipTestsCommand diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 2e443ed5ec..452dbaf0f0 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -54,9 +54,7 @@ object Dependencies { lazy val alpakkaSlick = "com.lightbend.akka" %% "akka-stream-alpakka-slick" % Version.alpakka lazy val postgres = "org.postgresql" % "postgresql" % Version.postgres lazy val h2 = "com.h2database" % "h2" % Version.h2 % Test - lazy val embeddedPg = - "ru.yandex.qatools.embed" % "postgresql-embedded" % Version.embeddedPg % Test exclude ("de.flapdoodle.embed", "de.flapdoodle.embed.process") - lazy val embeddedPgSupport = "de.flapdoodle.embed" % "de.flapdoodle.embed.process" % "2.1.2" % Test + lazy val testContainers = "org.testcontainers" % "testcontainers" % Version.testContainers % "test" lazy val s3Mock = "com.adobe.testing" % "s3mock" % "2.1.19" % Test lazy val apacheCommonsIO = "commons-io" % "commons-io" % "2.6" % Test lazy val keycloakAdapter = "org.keycloak" % "keycloak-adapter-core" % Version.keycloak diff --git a/project/Version.scala b/project/Version.scala index bab01b73e3..a79b75f32f 100644 --- a/project/Version.scala +++ b/project/Version.scala @@ -15,7 +15,6 @@ object Version { val slickPostgres = "0.21.1" val postgres = "42.6.0" val h2 = "2.1.214" - val embeddedPg = "2.10" val keycloak = "22.0.1" val resteasy = "6.2.4.Final" val jbossLogging = "3.5.1.Final" @@ -39,4 +38,5 @@ object Version { val diffx = "0.4.0" val log4j = "2.15.0" val kubernetesApi = "15.0.1" + val testContainers = "1.19.8" } \ No newline at end of file diff --git a/submission-errors/src/test/resources/application.conf b/submission-errors/src/test/resources/application.conf index cb28269578..c884943e7c 100644 --- a/submission-errors/src/test/resources/application.conf +++ b/submission-errors/src/test/resources/application.conf @@ -15,7 +15,7 @@ embedded-pg { profile = hmda.submissionerrors.repositories.PostgresEnhancedProfile url = "jdbc:postgresql://localhost:5432/postgres?user=postgres&password=postgres" numThreads = 2 - queueSize = 10 + queueSize = 100 connectionTimeout = 20000 validationTimeout = 10000 }