From fedd05115f087011c55399af40e56265f6628dbf Mon Sep 17 00:00:00 2001 From: PatrickHSI Date: Thu, 17 Sep 2020 13:27:37 -0400 Subject: [PATCH] endpoint to allow Admins to pull up submission files in HMDA Help --- .gitignore | 7 ++- .../scala/hmda/api/http/HmdaAdminApi.scala | 7 ++- .../http/admin/InstitutionAdminHttpApi.scala | 6 +-- .../http/admin/SubmissionAdminHttpApi.scala | 15 +++--- .../api/http/IntegrationQuarterlySpec.scala | 46 +++++++++++------- .../scala/hmda/api/http/IntegrationSpec.scala | 48 +++++++++++++------ 6 files changed, 82 insertions(+), 47 deletions(-) diff --git a/.gitignore b/.gitignore index b4a8675f04..f135b4ec14 100644 --- a/.gitignore +++ b/.gitignore @@ -84,9 +84,14 @@ src_managed/ project/boot/ project/plugins/project/ -#lightbend +## Lightbend .lightbend/ + +## Triplequote Hydra +.hydra/ + + ## Scala-IDE specific .scala_dependencies .worksheet diff --git a/hmda/src/main/scala/hmda/api/http/HmdaAdminApi.scala b/hmda/src/main/scala/hmda/api/http/HmdaAdminApi.scala index 2ef2d2f08e..005c78b3e5 100644 --- a/hmda/src/main/scala/hmda/api/http/HmdaAdminApi.scala +++ b/hmda/src/main/scala/hmda/api/http/HmdaAdminApi.scala @@ -36,11 +36,10 @@ object HmdaAdminApi { val shutdown = CoordinatedShutdown(system) val oAuth2Authorization = OAuth2Authorization(log, config) - val institutionRoutes = InstitutionAdminHttpApi.create(sharding, config) + val institutionRoutes = InstitutionAdminHttpApi.create(config,sharding) val publishRoutes = PublishAdminHttpApi.create(sharding, config) - val submissionRoutes = SubmissionAdminHttpApi.create(config, sharding, log) - val routes = BaseHttpApi.routes(name) ~ institutionRoutes(oAuth2Authorization) ~ submissionRoutes(oAuth2Authorization) - val routes = BaseHttpApi.routes(name) ~ institutionRoutes(oAuth2Authorization) ~ publishRoutes(oAuth2Authorization) + val submissionRoutes = SubmissionAdminHttpApi.create(log, config, sharding) + val routes = BaseHttpApi.routes(name) ~ institutionRoutes(oAuth2Authorization) ~ publishRoutes(oAuth2Authorization) ~ submissionRoutes(oAuth2Authorization) BaseHttpApi.runServer(shutdown, name)(timed(routes), host, port) Behaviors.empty diff --git a/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala b/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala index a60a1c7a91..b303b7d146 100644 --- a/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala +++ b/hmda/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala @@ -24,11 +24,11 @@ import scala.concurrent.{ ExecutionContext, Future } import scala.util.{ Failure, Success } object InstitutionAdminHttpApi { - def create(sharding: ClusterSharding, config: Config)(implicit ec: ExecutionContext, t: Timeout): OAuth2Authorization => Route = - new InstitutionAdminHttpApi(sharding, config)(ec, t).institutionAdminRoutes _ + def create(config: Config, sharding: ClusterSharding)(implicit ec: ExecutionContext, t: Timeout): OAuth2Authorization => Route = + new InstitutionAdminHttpApi(config, sharding)(ec, t).institutionAdminRoutes _ } -private class InstitutionAdminHttpApi(sharding: ClusterSharding, config: Config)(implicit ec: ExecutionContext, t: Timeout) { +private class InstitutionAdminHttpApi(config: Config, sharding: ClusterSharding)(implicit ec: ExecutionContext, t: Timeout) { val hmdaAdminRole = config.getString("keycloak.hmda.admin.role") val checkLEI = true val checkAgencyCode = false diff --git a/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala b/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala index 9ed360b4ac..0c447e5dbb 100644 --- a/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala +++ b/hmda/src/main/scala/hmda/api/http/admin/SubmissionAdminHttpApi.scala @@ -5,10 +5,10 @@ import akka.actor.typed.ActorSystem import akka.cluster.sharding.typed.scaladsl.ClusterSharding import akka.http.scaladsl.common.{CsvEntityStreamingSupport, EntityStreamingSupport} import akka.http.scaladsl.model.ContentTypes.`text/csv(UTF-8)` -import akka.http.scaladsl.model.HttpEntity -import akka.http.scaladsl.model.StatusCodes.{BadRequest, InternalServerError, NotFound, OK} +import akka.http.scaladsl.model.StatusCodes.{InternalServerError, OK} import akka.http.scaladsl.model.headers.ContentDispositionTypes.attachment import akka.http.scaladsl.model.headers.`Content-Disposition` +import akka.http.scaladsl.model.{HttpEntity, StatusCodes} import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server.Route import akka.stream.scaladsl.Source @@ -18,7 +18,6 @@ import cats.data.ValidatedNec import cats.implicits._ import com.typesafe.config.Config import hmda.api.http.admin.SubmissionAdminHttpApi.{pipeDelimitedFileStream, validateRawSubmissionId} -import hmda.api.http.model.ErrorResponse import hmda.auth.OAuth2Authorization import hmda.messages.submission.SubmissionCommands.GetSubmission import hmda.model.filing.lar.LoanApplicationRegister @@ -34,11 +33,11 @@ import org.slf4j.Logger import scala.util.{Failure, Success, Try} object SubmissionAdminHttpApi { - def create(config: Config, clusterSharding: ClusterSharding, log: Logger)( + def create(log: Logger, config: Config, clusterSharding: ClusterSharding)( implicit system: ActorSystem[_], timeout: Timeout ): OAuth2Authorization => Route = - new SubmissionAdminHttpApi(config, clusterSharding, log).routes + new SubmissionAdminHttpApi(log, config, clusterSharding).routes /** * Reads the existing file from the journal @@ -94,7 +93,7 @@ object SubmissionAdminHttpApi { } } -private class SubmissionAdminHttpApi(config: Config, clusterSharding: ClusterSharding, log: Logger)( +private class SubmissionAdminHttpApi(log: Logger, config: Config, clusterSharding: ClusterSharding)( implicit system: ActorSystem[_], timeout: Timeout ) { @@ -109,7 +108,7 @@ private class SubmissionAdminHttpApi(config: Config, clusterSharding: ClusterSha validateRawSubmissionId(rawSubmissionId) match { case Invalid(reason) => val formattedReasons = reason.mkString_(", ") - complete(BadRequest, ErrorResponse(BadRequest.intValue, formattedReasons, uri.path)) + complete((StatusCodes.BadRequest,formattedReasons)) case Valid(submissionId) => val submissionRef = SubmissionPersistence.selectSubmissionPersistence(clusterSharding, submissionId) @@ -120,7 +119,7 @@ private class SubmissionAdminHttpApi(config: Config, clusterSharding: ClusterSha complete(InternalServerError) case Success(None) => - complete(NotFound, ErrorResponse(NotFound.intValue, s"Submission with $submissionId does not exist", uri.path)) + complete((StatusCodes.NotFound,s"Submission with $submissionId does not exist")) case Success(Some(_)) => val csvSource = pipeDelimitedFileStream(submissionId).via(csvStreamingSupport.framingRenderer) diff --git a/hmda/src/test/scala/hmda/api/http/IntegrationQuarterlySpec.scala b/hmda/src/test/scala/hmda/api/http/IntegrationQuarterlySpec.scala index b81e36368d..8377fee1ec 100644 --- a/hmda/src/test/scala/hmda/api/http/IntegrationQuarterlySpec.scala +++ b/hmda/src/test/scala/hmda/api/http/IntegrationQuarterlySpec.scala @@ -3,23 +3,21 @@ package hmda.api.http import akka.actor.typed.ActorSystem import akka.actor.typed.scaladsl.adapter._ import akka.cluster.sharding.typed.scaladsl.ClusterSharding -import akka.cluster.typed.{ Cluster, Join } +import akka.cluster.typed.{Cluster, Join} import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.StatusCodes.Created import akka.http.scaladsl.testkit.ScalatestRouteTest -import akka.stream.scaladsl.{ Sink, Source } +import akka.stream.scaladsl.Sink import akka.util.Timeout -import hmda.api.http.admin.InstitutionAdminHttpApi +import com.typesafe.config.Config +import hmda.api.http.admin.{ InstitutionAdminHttpApi, SubmissionAdminHttpApi } import hmda.api.http.filing.submissions._ -import hmda.api.http.filing.{ FileUploadUtils, FilingHttpApi } -import hmda.api.http.model.filing.submissions.{ EditsSign, EditsVerification } -import hmda.auth.{ KeycloakTokenVerifier, OAuth2Authorization } -import hmda.messages.submission.SubmissionProcessingCommands.{ CompleteMacro, CompleteQuality, CompleteSyntacticalValidity } +import hmda.api.http.filing.{FileUploadUtils, FilingHttpApi} +import hmda.api.http.model.filing.submissions.{EditsSign, EditsVerification} +import hmda.auth.{KeycloakTokenVerifier, OAuth2Authorization} +import hmda.messages.submission.SubmissionProcessingCommands.{CompleteMacro, CompleteQuality, CompleteSyntacticalValidity} import hmda.model.filing.FilingDetails -import hmda.model.filing.lar.LarGenerators.larNGen -import hmda.model.filing.submission.{ Submission, SubmissionId } -import hmda.model.filing.ts.TransmittalSheet -import hmda.model.filing.ts.TsGenerators.tsGen +import hmda.model.filing.submission.{Submission, SubmissionId} import hmda.model.institution.Institution import hmda.model.institution.InstitutionGenerators.institutionGen import hmda.persistence.AkkaCassandraPersistenceSpec @@ -31,12 +29,12 @@ import io.circe.Encoder import io.circe.generic.semiauto._ import org.keycloak.adapters.KeycloakDeploymentBuilder import org.scalatest.MustMatchers -import org.scalatest.concurrent.Eventually -import org.scalatest.time.{ Millis, Minutes, Span } -import org.slf4j.{ Logger, LoggerFactory } +import org.scalatest.concurrent.{ Eventually, ScalaFutures } +import org.scalatest.time.{Millis, Minutes, Span} +import org.slf4j.{Logger, LoggerFactory} -import scala.concurrent.{ Await, ExecutionContext } import scala.concurrent.duration._ +import scala.concurrent.{Await, ExecutionContext} import scala.util.Random class IntegrationQuarterlySpec @@ -44,7 +42,8 @@ class IntegrationQuarterlySpec with MustMatchers with ScalatestRouteTest with FileUploadUtils - with Eventually { + with Eventually + with ScalaFutures { override implicit def patienceConfig: PatienceConfig = PatienceConfig(timeout = Span(2, Minutes), interval = Span(100, Millis)) @@ -54,8 +53,10 @@ class IntegrationQuarterlySpec implicit val timeout: Timeout = Timeout(duration) val sharding: ClusterSharding = ClusterSharding(typedSystem) val ec: ExecutionContext = system.dispatcher + val config: Config = system.settings.config - val institutionAdminRoute = InstitutionAdminHttpApi.create(sharding, system.settings.config) + val institutionAdminRoute = InstitutionAdminHttpApi.create(config, sharding) + val submissionAdminRoute = SubmissionAdminHttpApi.create(log, config, sharding) val filingRoute = FilingHttpApi.create(log, sharding) val submissionRoute = SubmissionHttpApi.create(log, sharding) val editsRoute = EditsHttpApi.create(log, sharding) @@ -151,6 +152,17 @@ class IntegrationQuarterlySpec responseAs[Submission] } + Get(s"/admin/hmdafile/${submissionQuarterly.id}") ~> submissionAdminRoute(oAuth2Authorization) ~> check { + status mustBe StatusCodes.OK + val futureLineCount = + response.entity.dataBytes + .via(FlowUtils.framing) + .map(_.utf8String) + .runWith(Sink.fold(0L)((acc, _) => acc + 1)) + + whenReady(futureLineCount)(actualLineCount => actualLineCount mustBe 1001) + } + val editsSummary = Get( s"/institutions/${sampleInstitutionQuarterly.LEI}/filings/${quarterlyPeriod.year}/quarter/${quarterlyPeriod.quarter.get}/submissions/${uploadFileSubmission.id.sequenceNumber}/edits" diff --git a/hmda/src/test/scala/hmda/api/http/IntegrationSpec.scala b/hmda/src/test/scala/hmda/api/http/IntegrationSpec.scala index 6ce5f2a085..cc86675ca9 100644 --- a/hmda/src/test/scala/hmda/api/http/IntegrationSpec.scala +++ b/hmda/src/test/scala/hmda/api/http/IntegrationSpec.scala @@ -3,21 +3,22 @@ package hmda.api.http import akka.actor.typed.ActorSystem import akka.actor.typed.scaladsl.adapter._ import akka.cluster.sharding.typed.scaladsl.ClusterSharding -import akka.cluster.typed.{ Cluster, Join } +import akka.cluster.typed.{Cluster, Join} import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.StatusCodes.Created -import akka.http.scaladsl.testkit.{ ScalatestRouteTest, WSProbe } +import akka.http.scaladsl.testkit.{ScalatestRouteTest, WSProbe} import akka.stream.scaladsl.Sink import akka.util.Timeout -import hmda.api.http.admin.InstitutionAdminHttpApi +import com.typesafe.config.Config +import hmda.api.http.admin.{ InstitutionAdminHttpApi, SubmissionAdminHttpApi } import hmda.api.http.filing.submissions._ -import hmda.api.http.filing.{ FileUploadUtils, FilingHttpApi } -import hmda.api.http.model.filing.submissions.{ EditsSign, EditsVerification } +import hmda.api.http.filing.{FileUploadUtils, FilingHttpApi} +import hmda.api.http.model.filing.submissions.{EditsSign, EditsVerification} import hmda.api.ws.filing.submissions.SubmissionWsApi -import hmda.auth.{ KeycloakTokenVerifier, OAuth2Authorization } -import hmda.messages.submission.SubmissionProcessingCommands.{ CompleteMacro, CompleteQuality, CompleteSyntacticalValidity } +import hmda.auth.{KeycloakTokenVerifier, OAuth2Authorization} +import hmda.messages.submission.SubmissionProcessingCommands.{CompleteMacro, CompleteQuality, CompleteSyntacticalValidity} import hmda.model.filing.FilingDetails -import hmda.model.filing.submission.{ Submission, SubmissionId } +import hmda.model.filing.submission.{Submission, SubmissionId} import hmda.model.institution.Institution import hmda.model.institution.InstitutionGenerators.institutionGen import hmda.persistence.AkkaCassandraPersistenceSpec @@ -29,15 +30,20 @@ import io.circe.Encoder import io.circe.generic.semiauto._ import org.keycloak.adapters.KeycloakDeploymentBuilder import org.scalatest.MustMatchers -import org.scalatest.concurrent.Eventually -import org.scalatest.time.{ Millis, Minutes, Span } -import org.slf4j.{ Logger, LoggerFactory } +import org.scalatest.concurrent.{Eventually, ScalaFutures} +import org.scalatest.time.{Millis, Minutes, Span} +import org.slf4j.{Logger, LoggerFactory} import scala.concurrent.duration._ -import scala.concurrent.{ Await, ExecutionContext } +import scala.concurrent.{Await, ExecutionContext} import scala.util.Random -class IntegrationSpec extends AkkaCassandraPersistenceSpec with MustMatchers with ScalatestRouteTest with FileUploadUtils with Eventually { +class IntegrationSpec extends AkkaCassandraPersistenceSpec + with MustMatchers + with ScalatestRouteTest + with FileUploadUtils + with Eventually + with ScalaFutures { override implicit def patienceConfig: PatienceConfig = PatienceConfig(timeout = Span(2, Minutes), interval = Span(100, Millis)) @@ -47,8 +53,11 @@ class IntegrationSpec extends AkkaCassandraPersistenceSpec with MustMatchers wit implicit val timeout: Timeout = Timeout(duration) val sharding: ClusterSharding = ClusterSharding(typedSystem) val ec: ExecutionContext = system.dispatcher + val config: Config = system.settings.config + + val institutionAdminRoute = InstitutionAdminHttpApi.create(config, sharding) + val submissionAdminRoute = SubmissionAdminHttpApi.create(log, config, sharding) - val institutionAdminRoute = InstitutionAdminHttpApi.create(sharding, system.settings.config) val filingRoute = FilingHttpApi.create(log, sharding) val submissionRoute = SubmissionHttpApi.create(log, sharding) val editsRoute = EditsHttpApi.create(log, sharding) @@ -127,6 +136,17 @@ class IntegrationSpec extends AkkaCassandraPersistenceSpec with MustMatchers wit responseAs[Submission] } + Get(s"/admin/hmdafile/${submissionYearly.id}") ~> submissionAdminRoute(oAuth2Authorization) ~> check { + status mustBe StatusCodes.OK + val futureLineCount = + response.entity.dataBytes + .via(FlowUtils.framing) + .map(_.utf8String) + .runWith(Sink.fold(0L)((acc, _) => acc + 1)) + + whenReady(futureLineCount)(actualLineCount => actualLineCount mustBe 1001) + } + val uploadFileSubmission = Post(s"/institutions/${sampleInstitution.LEI}/filings/$period/submissions/${submissionYearly.id.sequenceNumber}", hmdaFile) ~> fileUploadRoute( oAuth2Authorization