endpoint to allow Admins to pull up submission files in HMDA Help

cfpb · Sep 17, 2020 · fedd051 · fedd051
1 parent af17f8d
commit fedd051
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 47 deletions.
diff --git a/.gitignore b/.gitignore
@@ -84,9 +84,14 @@ src_managed/
 project/boot/
 project/plugins/project/
 
-#lightbend
+## Lightbend
 .lightbend/
 
+
+## Triplequote Hydra
+.hydra/
+
+
 ## Scala-IDE specific
 .scala_dependencies
 .worksheet

diff --git a/hmda/src/main/scala/hmda/api/http/HmdaAdminApi.scala b/hmda/src/main/scala/hmda/api/http/HmdaAdminApi.scala
@@ -36,11 +36,10 @@ object HmdaAdminApi {
     val shutdown                             = CoordinatedShutdown(system)
 
     val oAuth2Authorization = OAuth2Authorization(log, config)
-    val institutionRoutes   = InstitutionAdminHttpApi.create(sharding, config)
+    val institutionRoutes   = InstitutionAdminHttpApi.create(config,sharding)
     val publishRoutes       = PublishAdminHttpApi.create(sharding, config)
-    val submissionRoutes    = SubmissionAdminHttpApi.create(config, sharding, log)
-    val routes              = BaseHttpApi.routes(name) ~ institutionRoutes(oAuth2Authorization) ~ submissionRoutes(oAuth2Authorization)
-    val routes              = BaseHttpApi.routes(name) ~ institutionRoutes(oAuth2Authorization) ~ publishRoutes(oAuth2Authorization)
+    val submissionRoutes    = SubmissionAdminHttpApi.create(log, config, sharding)
+    val routes              = BaseHttpApi.routes(name) ~ institutionRoutes(oAuth2Authorization)  ~ publishRoutes(oAuth2Authorization) ~ submissionRoutes(oAuth2Authorization)
 
     BaseHttpApi.runServer(shutdown, name)(timed(routes), host, port)
     Behaviors.empty

diff --git a/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala b/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala
@@ -24,11 +24,11 @@ import scala.concurrent.{ ExecutionContext, Future }
 import scala.util.{ Failure, Success }
 
 object InstitutionAdminHttpApi {
-  def create(sharding: ClusterSharding, config: Config)(implicit ec: ExecutionContext, t: Timeout): OAuth2Authorization => Route =
-    new InstitutionAdminHttpApi(sharding, config)(ec, t).institutionAdminRoutes _
+  def create(config: Config, sharding: ClusterSharding)(implicit ec: ExecutionContext, t: Timeout): OAuth2Authorization => Route =
+    new InstitutionAdminHttpApi(config, sharding)(ec, t).institutionAdminRoutes _
 }
 
-private class InstitutionAdminHttpApi(sharding: ClusterSharding, config: Config)(implicit ec: ExecutionContext, t: Timeout) {
+private class InstitutionAdminHttpApi(config: Config, sharding: ClusterSharding)(implicit ec: ExecutionContext, t: Timeout) {
   val hmdaAdminRole   = config.getString("keycloak.hmda.admin.role")
   val checkLEI        = true
   val checkAgencyCode = false

diff --git a/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala b/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala
@@ -5,10 +5,10 @@ import akka.actor.typed.ActorSystem
 import akka.cluster.sharding.typed.scaladsl.ClusterSharding
 import akka.http.scaladsl.common.{CsvEntityStreamingSupport, EntityStreamingSupport}
 import akka.http.scaladsl.model.ContentTypes.`text/csv(UTF-8)`
-import akka.http.scaladsl.model.HttpEntity
-import akka.http.scaladsl.model.StatusCodes.{BadRequest, InternalServerError, NotFound, OK}
+import akka.http.scaladsl.model.StatusCodes.{InternalServerError, OK}
 import akka.http.scaladsl.model.headers.ContentDispositionTypes.attachment
 import akka.http.scaladsl.model.headers.`Content-Disposition`
+import akka.http.scaladsl.model.{HttpEntity, StatusCodes}
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.Route
 import akka.stream.scaladsl.Source
@@ -18,7 +18,6 @@ import cats.data.ValidatedNec
 import cats.implicits._
 import com.typesafe.config.Config
 import hmda.api.http.admin.SubmissionAdminHttpApi.{pipeDelimitedFileStream, validateRawSubmissionId}
-import hmda.api.http.model.ErrorResponse
 import hmda.auth.OAuth2Authorization
 import hmda.messages.submission.SubmissionCommands.GetSubmission
 import hmda.model.filing.lar.LoanApplicationRegister
@@ -34,11 +33,11 @@ import org.slf4j.Logger
 import scala.util.{Failure, Success, Try}
 
 object SubmissionAdminHttpApi {
-  def create(config: Config, clusterSharding: ClusterSharding, log: Logger)(
+  def create(log: Logger, config: Config, clusterSharding: ClusterSharding)(
     implicit system: ActorSystem[_],
     timeout: Timeout
   ): OAuth2Authorization => Route =
-    new SubmissionAdminHttpApi(config, clusterSharding, log).routes
+    new SubmissionAdminHttpApi(log, config, clusterSharding).routes
 
   /**
    * Reads the existing file from the journal
@@ -94,7 +93,7 @@ object SubmissionAdminHttpApi {
   }
 }
 
-private class SubmissionAdminHttpApi(config: Config, clusterSharding: ClusterSharding, log: Logger)(
+private class SubmissionAdminHttpApi(log: Logger, config: Config, clusterSharding: ClusterSharding)(
   implicit system: ActorSystem[_],
   timeout: Timeout
 ) {
@@ -109,7 +108,7 @@ private class SubmissionAdminHttpApi(config: Config, clusterSharding: ClusterSha
         validateRawSubmissionId(rawSubmissionId) match {
           case Invalid(reason) =>
             val formattedReasons = reason.mkString_(", ")
-            complete(BadRequest, ErrorResponse(BadRequest.intValue, formattedReasons, uri.path))
+            complete((StatusCodes.BadRequest,formattedReasons))
 
           case Valid(submissionId) =>
             val submissionRef    = SubmissionPersistence.selectSubmissionPersistence(clusterSharding, submissionId)
@@ -120,7 +119,7 @@ private class SubmissionAdminHttpApi(config: Config, clusterSharding: ClusterSha
                 complete(InternalServerError)
 
               case Success(None) =>
-                complete(NotFound, ErrorResponse(NotFound.intValue, s"Submission with $submissionId does not exist", uri.path))
+                complete((StatusCodes.NotFound,s"Submission with $submissionId does not exist"))
 
               case Success(Some(_)) =>
                 val csvSource = pipeDelimitedFileStream(submissionId).via(csvStreamingSupport.framingRenderer)

diff --git a/hmda/src/test/scala/hmda/api/http/IntegrationQuarterlySpec.scala b/hmda/src/test/scala/hmda/api/http/IntegrationQuarterlySpec.scala
@@ -3,23 +3,21 @@ package hmda.api.http
 import akka.actor.typed.ActorSystem
 import akka.actor.typed.scaladsl.adapter._
 import akka.cluster.sharding.typed.scaladsl.ClusterSharding
-import akka.cluster.typed.{ Cluster, Join }
+import akka.cluster.typed.{Cluster, Join}
 import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.model.StatusCodes.Created
 import akka.http.scaladsl.testkit.ScalatestRouteTest
-import akka.stream.scaladsl.{ Sink, Source }
+import akka.stream.scaladsl.Sink
 import akka.util.Timeout
-import hmda.api.http.admin.InstitutionAdminHttpApi
+import com.typesafe.config.Config
+import hmda.api.http.admin.{ InstitutionAdminHttpApi, SubmissionAdminHttpApi }
 import hmda.api.http.filing.submissions._
-import hmda.api.http.filing.{ FileUploadUtils, FilingHttpApi }
-import hmda.api.http.model.filing.submissions.{ EditsSign, EditsVerification }
-import hmda.auth.{ KeycloakTokenVerifier, OAuth2Authorization }
-import hmda.messages.submission.SubmissionProcessingCommands.{ CompleteMacro, CompleteQuality, CompleteSyntacticalValidity }
+import hmda.api.http.filing.{FileUploadUtils, FilingHttpApi}
+import hmda.api.http.model.filing.submissions.{EditsSign, EditsVerification}
+import hmda.auth.{KeycloakTokenVerifier, OAuth2Authorization}
+import hmda.messages.submission.SubmissionProcessingCommands.{CompleteMacro, CompleteQuality, CompleteSyntacticalValidity}
 import hmda.model.filing.FilingDetails
-import hmda.model.filing.lar.LarGenerators.larNGen
-import hmda.model.filing.submission.{ Submission, SubmissionId }
-import hmda.model.filing.ts.TransmittalSheet
-import hmda.model.filing.ts.TsGenerators.tsGen
+import hmda.model.filing.submission.{Submission, SubmissionId}
 import hmda.model.institution.Institution
 import hmda.model.institution.InstitutionGenerators.institutionGen
 import hmda.persistence.AkkaCassandraPersistenceSpec
@@ -31,20 +29,21 @@ import io.circe.Encoder
 import io.circe.generic.semiauto._
 import org.keycloak.adapters.KeycloakDeploymentBuilder
 import org.scalatest.MustMatchers
-import org.scalatest.concurrent.Eventually
-import org.scalatest.time.{ Millis, Minutes, Span }
-import org.slf4j.{ Logger, LoggerFactory }
+import org.scalatest.concurrent.{ Eventually, ScalaFutures }
+import org.scalatest.time.{Millis, Minutes, Span}
+import org.slf4j.{Logger, LoggerFactory}
 
-import scala.concurrent.{ Await, ExecutionContext }
 import scala.concurrent.duration._
+import scala.concurrent.{Await, ExecutionContext}
 import scala.util.Random
 
 class IntegrationQuarterlySpec
   extends AkkaCassandraPersistenceSpec
     with MustMatchers
     with ScalatestRouteTest
     with FileUploadUtils
-    with Eventually {
+    with Eventually
+    with ScalaFutures {
 
   override implicit def patienceConfig: PatienceConfig = PatienceConfig(timeout = Span(2, Minutes), interval = Span(100, Millis))
 
@@ -54,8 +53,10 @@ class IntegrationQuarterlySpec
   implicit val timeout: Timeout            = Timeout(duration)
   val sharding: ClusterSharding            = ClusterSharding(typedSystem)
   val ec: ExecutionContext                 = system.dispatcher
+  val config: Config                       = system.settings.config
 
-  val institutionAdminRoute = InstitutionAdminHttpApi.create(sharding, system.settings.config)
+  val institutionAdminRoute = InstitutionAdminHttpApi.create(config, sharding)
+  val submissionAdminRoute  = SubmissionAdminHttpApi.create(log, config, sharding)
   val filingRoute           = FilingHttpApi.create(log, sharding)
   val submissionRoute       = SubmissionHttpApi.create(log, sharding)
   val editsRoute            = EditsHttpApi.create(log, sharding)
@@ -151,6 +152,17 @@ class IntegrationQuarterlySpec
           responseAs[Submission]
         }
 
+      Get(s"/admin/hmdafile/${submissionQuarterly.id}") ~> submissionAdminRoute(oAuth2Authorization) ~> check {
+        status mustBe StatusCodes.OK
+        val futureLineCount =
+          response.entity.dataBytes
+            .via(FlowUtils.framing)
+            .map(_.utf8String)
+            .runWith(Sink.fold(0L)((acc, _) => acc + 1))
+
+        whenReady(futureLineCount)(actualLineCount => actualLineCount mustBe 1001)
+      }
+
       val editsSummary =
         Get(
           s"/institutions/${sampleInstitutionQuarterly.LEI}/filings/${quarterlyPeriod.year}/quarter/${quarterlyPeriod.quarter.get}/submissions/${uploadFileSubmission.id.sequenceNumber}/edits"

diff --git a/hmda/src/test/scala/hmda/api/http/IntegrationSpec.scala b/hmda/src/test/scala/hmda/api/http/IntegrationSpec.scala
@@ -3,21 +3,22 @@ package hmda.api.http
 import akka.actor.typed.ActorSystem
 import akka.actor.typed.scaladsl.adapter._
 import akka.cluster.sharding.typed.scaladsl.ClusterSharding
-import akka.cluster.typed.{ Cluster, Join }
+import akka.cluster.typed.{Cluster, Join}
 import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.model.StatusCodes.Created
-import akka.http.scaladsl.testkit.{ ScalatestRouteTest, WSProbe }
+import akka.http.scaladsl.testkit.{ScalatestRouteTest, WSProbe}
 import akka.stream.scaladsl.Sink
 import akka.util.Timeout
-import hmda.api.http.admin.InstitutionAdminHttpApi
+import com.typesafe.config.Config
+import hmda.api.http.admin.{ InstitutionAdminHttpApi, SubmissionAdminHttpApi }
 import hmda.api.http.filing.submissions._
-import hmda.api.http.filing.{ FileUploadUtils, FilingHttpApi }
-import hmda.api.http.model.filing.submissions.{ EditsSign, EditsVerification }
+import hmda.api.http.filing.{FileUploadUtils, FilingHttpApi}
+import hmda.api.http.model.filing.submissions.{EditsSign, EditsVerification}
 import hmda.api.ws.filing.submissions.SubmissionWsApi
-import hmda.auth.{ KeycloakTokenVerifier, OAuth2Authorization }
-import hmda.messages.submission.SubmissionProcessingCommands.{ CompleteMacro, CompleteQuality, CompleteSyntacticalValidity }
+import hmda.auth.{KeycloakTokenVerifier, OAuth2Authorization}
+import hmda.messages.submission.SubmissionProcessingCommands.{CompleteMacro, CompleteQuality, CompleteSyntacticalValidity}
 import hmda.model.filing.FilingDetails
-import hmda.model.filing.submission.{ Submission, SubmissionId }
+import hmda.model.filing.submission.{Submission, SubmissionId}
 import hmda.model.institution.Institution
 import hmda.model.institution.InstitutionGenerators.institutionGen
 import hmda.persistence.AkkaCassandraPersistenceSpec
@@ -29,15 +30,20 @@ import io.circe.Encoder
 import io.circe.generic.semiauto._
 import org.keycloak.adapters.KeycloakDeploymentBuilder
 import org.scalatest.MustMatchers
-import org.scalatest.concurrent.Eventually
-import org.scalatest.time.{ Millis, Minutes, Span }
-import org.slf4j.{ Logger, LoggerFactory }
+import org.scalatest.concurrent.{Eventually, ScalaFutures}
+import org.scalatest.time.{Millis, Minutes, Span}
+import org.slf4j.{Logger, LoggerFactory}
 
 import scala.concurrent.duration._
-import scala.concurrent.{ Await, ExecutionContext }
+import scala.concurrent.{Await, ExecutionContext}
 import scala.util.Random
 
-class IntegrationSpec extends AkkaCassandraPersistenceSpec with MustMatchers with ScalatestRouteTest with FileUploadUtils with Eventually {
+class IntegrationSpec   extends AkkaCassandraPersistenceSpec
+  with MustMatchers
+  with ScalatestRouteTest
+  with FileUploadUtils
+  with Eventually
+  with ScalaFutures {
 
   override implicit def patienceConfig: PatienceConfig = PatienceConfig(timeout = Span(2, Minutes), interval = Span(100, Millis))
 
@@ -47,8 +53,11 @@ class IntegrationSpec extends AkkaCassandraPersistenceSpec with MustMatchers wit
   implicit val timeout: Timeout            = Timeout(duration)
   val sharding: ClusterSharding            = ClusterSharding(typedSystem)
   val ec: ExecutionContext                 = system.dispatcher
+  val config: Config                       = system.settings.config
+
+  val institutionAdminRoute = InstitutionAdminHttpApi.create(config, sharding)
+  val submissionAdminRoute  = SubmissionAdminHttpApi.create(log, config, sharding)
 
-  val institutionAdminRoute = InstitutionAdminHttpApi.create(sharding, system.settings.config)
   val filingRoute           = FilingHttpApi.create(log, sharding)
   val submissionRoute       = SubmissionHttpApi.create(log, sharding)
   val editsRoute            = EditsHttpApi.create(log, sharding)
@@ -127,6 +136,17 @@ class IntegrationSpec extends AkkaCassandraPersistenceSpec with MustMatchers wit
           responseAs[Submission]
         }
 
+      Get(s"/admin/hmdafile/${submissionYearly.id}") ~> submissionAdminRoute(oAuth2Authorization) ~> check {
+        status mustBe StatusCodes.OK
+        val futureLineCount =
+          response.entity.dataBytes
+            .via(FlowUtils.framing)
+            .map(_.utf8String)
+            .runWith(Sink.fold(0L)((acc, _) => acc + 1))
+
+        whenReady(futureLineCount)(actualLineCount => actualLineCount mustBe 1001)
+      }
+
       val uploadFileSubmission =
         Post(s"/institutions/${sampleInstitution.LEI}/filings/$period/submissions/${submissionYearly.id.sequenceNumber}", hmdaFile) ~> fileUploadRoute(
           oAuth2Authorization