finalize

cfpb · Sep 18, 2020 · 3d0d160 · 3d0d160
1 parent b348f23
commit 3d0d160
Show file tree

Hide file tree

Showing 13 changed files with 196 additions and 64 deletions.
diff --git a/build.sbt b/build.sbt
@@ -29,13 +29,9 @@ lazy val akkaDeps = Seq(
   akkaStreamsTestKit,
   akkaCors,
   akkaKafkaStreams,
-  embeddedKafka
-)
-
-lazy val pubDeps = Seq(
-  akkaQuartzScheduler,
+  embeddedKafka,
   alpakkaS3,
-  alpakkaFile
+  akkaQuartzScheduler
 )
 
 lazy val akkaPersistenceDeps =
@@ -54,7 +50,7 @@ lazy val akkaHttpDeps =
 lazy val circeDeps      = Seq(circe, circeGeneric, circeParser)
 lazy val enumeratumDeps = Seq(enumeratum, enumeratumCirce)
 
-lazy val slickDeps = Seq(slick, slickHikaryCP, postgres, h2)
+lazy val slickDeps = Seq(slick, slickHikariCP, postgres, h2)
 
 lazy val dockerSettings = Seq(
   Docker / maintainer := "Hmda-Ops",
@@ -102,11 +98,12 @@ lazy val common = (project in file("common"))
       scalapb.gen() -> (sourceManaged in Compile).value
     ),
     Seq(
-      libraryDependencies ++= commonDeps ++ authDeps ++ akkaDeps ++ akkaPersistenceDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ List(cormorant)
+      libraryDependencies ++= commonDeps ++ authDeps ++ akkaDeps ++ akkaPersistenceDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++ List(
+        cormorant
+      )
     )
   )
 
-
 lazy val `hmda-platform` = (project in file("hmda"))
   .enablePlugins(
     JavaServerAppPackaging,
@@ -124,9 +121,9 @@ lazy val `hmda-platform` = (project in file("hmda"))
         case "cinnamon-reference.conf"               => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
         case "logback.xml"                           => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -154,9 +151,9 @@ lazy val `check-digit` = (project in file("check-digit"))
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -177,19 +174,17 @@ lazy val `institutions-api` = (project in file("institutions-api"))
   .settings(hmdaBuildSettings: _*)
   .settings(
     Seq(
-      libraryDependencies ++= pubDeps,
       mainClass in Compile := Some("hmda.institution.HmdaInstitutionApi"),
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
       },
-      libraryDependencies ++= commonDeps,
       assemblyJarName in assembly := {
         s"${name.value}.jar"
       }
@@ -208,17 +203,16 @@ lazy val `hmda-data-publisher` = (project in file("hmda-data-publisher"))
   .settings(hmdaBuildSettings: _*)
   .settings(
     Seq(
-      libraryDependencies ++= pubDeps,
       mainClass in Compile := Some("hmda.publisher.HmdaDataPublisherApp"),
       assemblyJarName in assembly := {
         s"${name.value}.jar"
       },
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -244,9 +238,9 @@ lazy val `hmda-dashboard` = (project in file("hmda-dashboard"))
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -270,14 +264,13 @@ lazy val `ratespread-calculator` = (project in file("ratespread-calculator"))
   .settings(hmdaBuildSettings: _*)
   .settings(
     Seq(
-      libraryDependencies ++= pubDeps,
       mainClass in Compile := Some("hmda.calculator.HmdaRateSpread"),
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -301,14 +294,13 @@ lazy val `modified-lar` = (project in file("modified-lar"))
   .settings(hmdaBuildSettings: _*)
   .settings(
     Seq(
-      libraryDependencies ++= pubDeps,
       mainClass in Compile := Some("hmda.publication.lar.ModifiedLarApp"),
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -337,9 +329,9 @@ lazy val `irs-publisher` = (project in file("irs-publisher"))
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -368,9 +360,9 @@ lazy val `hmda-reporting` = (project in file("hmda-reporting"))
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -408,9 +400,9 @@ lazy val `hmda-analytics` = (project in file("hmda-analytics"))
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -438,9 +430,9 @@ lazy val `rate-limit` = (project in file("rate-limit"))
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -452,6 +444,7 @@ lazy val `rate-limit` = (project in file("rate-limit"))
     dockerSettings,
     packageSettings
   )
+  .dependsOn(common % "compile->compile;test->test")
   .dependsOn(`hmda-protocol`)
 
 lazy val `data-browser` = (project in file("data-browser"))
@@ -463,14 +456,14 @@ lazy val `data-browser` = (project in file("data-browser"))
   .settings(hmdaBuildSettings: _*)
   .settings(
     Seq(
-      libraryDependencies ++=  pubDeps ++
+      libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++
         enumeratumDeps :+ monix :+ lettuce :+ scalaJava8Compat :+ scalaMock,
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)
@@ -488,12 +481,11 @@ lazy val `submission-errors` = (project in file("submission-errors"))
   .enablePlugins(JavaServerAppPackaging, sbtdocker.DockerPlugin, AshScriptPlugin)
   .settings(hmdaBuildSettings)
   .settings(
-    libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps ++
-      enumeratumDeps :+ monix,
+    libraryDependencies ++= commonDeps ++ akkaDeps ++ akkaHttpDeps ++ circeDeps ++ slickDeps :+ monix :+ slickPostgres,
     dockerSettings,
     packageSettings
   )
-  .dependsOn(common)
+  .dependsOn(common % "compile->compile;test->test") // allows you to depend on code from both compile and test scopes
 
 lazy val `email-service` = (project in file("email-service"))
   .enablePlugins(JavaServerAppPackaging, sbtdocker.DockerPlugin, AshScriptPlugin)
@@ -504,9 +496,9 @@ lazy val `email-service` = (project in file("email-service"))
       assemblyMergeStrategy in assembly := {
         case "application.conf"                      => MergeStrategy.concat
         case "META-INF/io.netty.versions.properties" => MergeStrategy.concat
-        case PathList(ps @ _ *) if ps.last endsWith ".proto" =>
+        case PathList(ps @ _*) if ps.last endsWith ".proto" =>
           MergeStrategy.first
-        case "module-info.class"                     => MergeStrategy.concat
+        case "module-info.class" => MergeStrategy.concat
         case x =>
           val oldStrategy = (assemblyMergeStrategy in assembly).value
           oldStrategy(x)

diff --git a/common/src/test/scala/hmda/utils/EmbeddedPostgres.scala b/common/src/test/scala/hmda/utils/EmbeddedPostgres.scala
@@ -17,7 +17,8 @@ import scala.concurrent.ExecutionContext.Implicits.global
  */
 trait EmbeddedPostgres extends BeforeAndAfterAll with BeforeAndAfterEach { self: Suite =>
   private val embeddedPg = new ru.yandex.qatools.embed.postgresql.EmbeddedPostgres(Version.V10_6)
-  val dbConfig           = DatabaseConfig.forConfig[JdbcProfile]("embedded-pg")
+  val dbHoconpath        = "embedded-pg"
+  val dbConfig           = DatabaseConfig.forConfig[JdbcProfile](dbHoconpath)
 
   def bootstrapSqlFile: String
 

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -29,6 +29,7 @@ object Dependencies {
   lazy val akkaHttp2                = "com.typesafe.akka" %% "akka-http2-support"          % Version.akkaHttp
   lazy val akkaHttpTestkit          = "com.typesafe.akka" %% "akka-http-testkit"           % Version.akkaHttp % Test
   lazy val akkaHttpSprayJson        = "com.typesafe.akka" %% "akka-http-spray-json"        % Version.akkaHttp
+  lazy val slickPostgres            = "com.github.tminglei" %% "slick-pg"                  % Version.slickPostgres
   lazy val akkaHttpXml              = "com.typesafe.akka" %% "akka-http-xml"               % Version.akkaHttp
   lazy val akkaHttpCirce            = "de.heikoseeberger" %% "akka-http-circe"             % Version.akkaHttpJson
   lazy val akkaManagementClusterBootstrap =
@@ -49,7 +50,7 @@ object Dependencies {
   lazy val akkaPersistenceCassandra = "com.typesafe.akka"  %% "akka-persistence-cassandra"          % Version.cassandraPluginVersion
   lazy val cassandraLauncher        = "com.typesafe.akka"  %% "akka-persistence-cassandra-launcher" % Version.cassandraPluginVersion
   lazy val slick                    = "com.typesafe.slick" %% "slick"                               % Version.slick
-  lazy val slickHikaryCP            = "com.typesafe.slick" %% "slick-hikaricp"                      % Version.slick
+  lazy val slickHikariCP            = "com.typesafe.slick" %% "slick-hikaricp"                      % Version.slick
   lazy val alpakkaSlick             = "com.lightbend.akka" %% "akka-stream-alpakka-slick"           % Version.alpakkaSlick
   lazy val postgres                 = "org.postgresql"     % "postgresql"                           % Version.postgres
   lazy val h2                       = "com.h2database"     % "h2"                                   % Version.h2 % Test

diff --git a/project/Version.scala b/project/Version.scala
@@ -12,6 +12,7 @@ object Version {
   val cassandraPluginVersion = "0.101"
   val akkaClusterDowning     = "0.0.12"
   val slick                  = "3.3.2"
+  val slickPostgres          = "0.19.2"
   val postgres               = "42.2.5"
   val h2                     = "1.4.200"
   val embeddedPg             = "2.10"

diff --git a/submission-errors/src/main/resources/application.conf b/submission-errors/src/main/resources/application.conf
@@ -50,10 +50,9 @@ dbconfig {
 }
 
 submission-errors-db {
-  profile = "slick.jdbc.PostgresProfile$"
-
+  profile = "hmda.submissionerrors.repositories.PostgresEnhancedProfile$"
   db {
-    profile = slick.jdbc.PostgresProfile
+    profile = hmda.submissionerrors.repositories.PostgresEnhancedProfile
     url = "jdbc:postgresql://"${dbconfig.host}":"${dbconfig.port}"/"${dbconfig.database}"?user="${dbconfig.user}"&password="${dbconfig.password}"&sslmode="${dbconfig.ssl}"&"${dbconfig.additionalConfig}
     url = ${?JDBC_URL}
     numThreads = 8

diff --git a/submission-errors/src/main/scala/hmda/submissionerrors/SubmissionErrorsApp.scala b/submission-errors/src/main/scala/hmda/submissionerrors/SubmissionErrorsApp.scala
@@ -14,21 +14,19 @@ import hmda.submissionerrors.streams.SubmissionProcessor.{ handleMessages, proce
 import monix.execution.Scheduler
 import org.apache.kafka.clients.consumer.ConsumerConfig
 import org.apache.kafka.common.serialization.StringDeserializer
-import slick.basic.DatabaseConfig
-import slick.jdbc.JdbcProfile
 
 object SubmissionErrorsApp extends App {
   val config         = ConfigFactory.load()
   val kafkaHosts     = config.getString("kafka.hosts")
   val kafkaTopic     = config.getString("kafka.topic")
   val databaseTable  = config.getString("dbconfig.table")
-  val databaseConfig = DatabaseConfig.forConfig[JdbcProfile]("submission-errors-db")
+  val databaseConfig = PostgresSubmissionErrorRepository.config("submission-errors-db")
 
   val classicSystem: ClassicActorSystem    = ClassicActorSystem("submission-errors-app", config)
   implicit val typedSystem: ActorSystem[_] = classicSystem.toTyped
   implicit val monixScheduler: Scheduler   = Scheduler(classicSystem.toTyped.executionContext)
 
-  val repo = new PostgresSubmissionErrorRepository(databaseConfig, databaseTable)
+  val repo = PostgresSubmissionErrorRepository.make(databaseConfig, databaseTable)
 
   val kafkaConsumerSettings: ConsumerSettings[String, String] =
     ConsumerSettings(

diff --git a/...on-errors/src/main/scala/hmda/submissionerrors/repositories/PostgresEnhancedProfile.scala b/...on-errors/src/main/scala/hmda/submissionerrors/repositories/PostgresEnhancedProfile.scala
@@ -0,0 +1,18 @@
+package hmda.submissionerrors.repositories
+
+import com.github.tminglei.slickpg.{ ExPostgresProfile, PgArraySupport }
+import slick.basic.Capability
+
+trait PostgresEnhancedProfile extends ExPostgresProfile with PgArraySupport {
+  override protected def computeCapabilities: Set[Capability] =
+    super.computeCapabilities + slick.jdbc.JdbcCapabilities.insertOrUpdate
+
+  override val api: API = new API {}
+
+  // NOTE: You need to use the super.API syntax otherwise the implicits won't resolve correctly
+  trait API extends super.API with ArrayImplicits {
+    implicit val simpleStrVectorTypeMapper: DriverJdbcType[Vector[String]] =
+      new SimpleArrayJdbcType[String]("text").to(_.toVector)
+  }
+}
+object PostgresEnhancedProfile extends PostgresEnhancedProfile