Skip to content

Commit ce09245

Browse files
agilelab-tmnd1991agilelab-tmnd1991
committed
[#548] Run tests on Apple Silicon
# New features and improvements Test suites pass on Apple Silicon. # Breaking changes None. # Migration None. # Bug fixes None. # How this feature was tested Existing tests: before and after same tests execute on Gitlab CI. # Related issue Closes #548
1 parent 0dfebbb commit ce09245

28 files changed

+143
-78
lines changed

‎project/CDP717Dependencies.scala‎

Lines changed: 34 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,8 @@ class CDP717Dependencies(versions: CDP717Versions) extends Dependencies {
1212
case "2.11" => ("it.agilelab.bigdata.spark" % "spark-solr" % versions.sparkSolr)
1313
case "2.12" => ("it.agilelab.bigdata.spark" %% "spark-solr" % versions.sparkSolr)
1414
}
15-
lazy val shapeless = "com.chuusai" %% "shapeless" % "2.3.3"
16-
lazy val javaxMail = "javax.mail" % "mail" % "1.4"
15+
lazy val shapeless = "com.chuusai" %% "shapeless" % "2.3.3"
16+
lazy val javaxMail = "javax.mail" % "mail" % "1.4"
1717
lazy val globalExclusions: Seq[ExclusionRule] = Seq(
1818
ExclusionRule("org.apache.logging.log4j", "log4j-slf4j-impl"),
1919
ExclusionRule("com.fasterxml.jackson.module", "jackson-module-scala_2.12"),
@@ -505,26 +505,26 @@ class CDP717Dependencies(versions: CDP717Versions) extends Dependencies {
505505
("org.apache.avro" % "avro-mapred" % "1.8.2.7.1.7.0-551").classifier("hadoop2"),
506506
("org.apache.avro" % "trevni-avro" % "1.8.2.7.1.7.0-551").classifier("hadoop2")
507507
)
508-
lazy val akkaActor = "com.typesafe.akka" %% "akka-actor" % versions.akka
509-
lazy val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % versions.akka
510-
lazy val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % versions.akka
511-
lazy val akkaContrib = "com.typesafe.akka" %% "akka-contrib" % versions.akka
512-
lazy val akkaHttp = "com.typesafe.akka" %% "akka-http" % versions.akkaHttp
513-
lazy val akkaHttpSpray = "com.typesafe.akka" %% "akka-http-spray-json" % versions.akkaHttp
514-
lazy val akkaKryo = "com.github.romix.akka" %% "akka-kryo-serialization" % "0.5.0"
515-
lazy val akkaRemote = "com.typesafe.akka" %% "akka-remote" % versions.akka
516-
lazy val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % versions.akka
517-
lazy val akkaStream = "com.typesafe.akka" %% "akka-stream" % versions.akka
518-
lazy val akkaStreamTestkit = "com.typesafe.akka" %% "akka-stream-testkit" % versions.akka % Test
519-
lazy val akkaHttpTestKit = "com.typesafe.akka" %% "akka-http-testkit" % versions.akkaHttp % Test
520-
lazy val akkaClusterTestKit = "com.typesafe.akka" %% "akka-multi-node-testkit" % versions.akka % Test
521-
lazy val akkaTestKit = "com.typesafe.akka" %% "akka-testkit" % versions.akka % Test
522-
lazy val sttpCore = "com.softwaremill.sttp.client" %% "core" % versions.sttpVersion
523-
lazy val sttpJson4s = "com.softwaremill.sttp.client" %% "json4s" % versions.sttpVersion
524-
lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % versions.json4s
525-
lazy val mongodbScala = "org.mongodb.scala" %% "mongo-scala-driver" % versions.mongodbScala
526-
lazy val mongoTest = "com.github.simplyscala" %% "scalatest-embedmongo" % "0.2.4" % Test
527-
lazy val scalaTest = "org.scalatest" %% "scalatest" % versions.scalaTest % Test
508+
lazy val akkaActor = "com.typesafe.akka" %% "akka-actor" % versions.akka
509+
lazy val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % versions.akka
510+
lazy val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % versions.akka
511+
lazy val akkaContrib = "com.typesafe.akka" %% "akka-contrib" % versions.akka
512+
lazy val akkaHttp = "com.typesafe.akka" %% "akka-http" % versions.akkaHttp
513+
lazy val akkaHttpSpray = "com.typesafe.akka" %% "akka-http-spray-json" % versions.akkaHttp
514+
lazy val akkaKryo = "com.github.romix.akka" %% "akka-kryo-serialization" % "0.5.0"
515+
lazy val akkaRemote = "com.typesafe.akka" %% "akka-remote" % versions.akka
516+
lazy val akkaSlf4j = "com.typesafe.akka" %% "akka-slf4j" % versions.akka
517+
lazy val akkaStream = "com.typesafe.akka" %% "akka-stream" % versions.akka
518+
lazy val akkaStreamTestkit = "com.typesafe.akka" %% "akka-stream-testkit" % versions.akka % Test
519+
lazy val akkaHttpTestKit = "com.typesafe.akka" %% "akka-http-testkit" % versions.akkaHttp % Test
520+
lazy val akkaClusterTestKit = "com.typesafe.akka" %% "akka-multi-node-testkit" % versions.akka % Test
521+
lazy val akkaTestKit = "com.typesafe.akka" %% "akka-testkit" % versions.akka % Test
522+
lazy val sttpCore = "com.softwaremill.sttp.client" %% "core" % versions.sttpVersion
523+
lazy val sttpJson4s = "com.softwaremill.sttp.client" %% "json4s" % versions.sttpVersion
524+
lazy val json4sJackson = "org.json4s" %% "json4s-jackson" % versions.json4s
525+
lazy val mongodbScala = "org.mongodb.scala" %% "mongo-scala-driver" % versions.mongodbScala
526+
lazy val mongoTest = "de.flapdoodle.embed" % "de.flapdoodle.embed.mongo" % "3.4.6" % Test
527+
lazy val scalaTest = "org.scalatest" %% "scalatest" % versions.scalaTest % Test
528528
lazy val allAkka = Seq(
529529
akkaActor,
530530
akkaCluster,
@@ -586,17 +586,19 @@ class CDP717Dependencies(versions: CDP717Versions) extends Dependencies {
586586
override lazy val modelDependencies
587587
: Seq[sbt.ModuleID] = parcelDependencies ++ allAkka ++ testDependencies :+ mongoTest :+ mongodbScala
588588
override lazy val coreDependencies
589-
: Seq[sbt.ModuleID] = Seq(darwinCore, darwinMockConnector) ++ avro4s ++ testDependencies
590-
override lazy val testDependencies: Seq[sbt.ModuleID] = Seq(scalaTest)
591-
override lazy val scalaTestDependencies: Seq[sbt.ModuleID] = testDependencies
592-
override lazy val repositoryCoreDependencies: Seq[sbt.ModuleID] = testDependencies ++ Seq(shapeless)
593-
override lazy val repositoryMongoDependencies: Seq[sbt.ModuleID] = Seq(nameOf, mongoTest) ++ testDependencies ++ Seq(shapeless)
589+
: Seq[sbt.ModuleID] = Seq(darwinCore, darwinMockConnector) ++ avro4s ++ testDependencies
590+
override lazy val testDependencies: Seq[sbt.ModuleID] = Seq(scalaTest)
591+
override lazy val scalaTestDependencies: Seq[sbt.ModuleID] = testDependencies
592+
override lazy val repositoryCoreDependencies: Seq[sbt.ModuleID] = testDependencies ++ Seq(shapeless)
593+
override lazy val repositoryMongoDependencies: Seq[sbt.ModuleID] = Seq(nameOf, mongoTest) ++ testDependencies ++ Seq(
594+
shapeless
595+
)
594596
override lazy val repositoryPostgresDependencies: Seq[sbt.ModuleID] = Seq(postgresqlEmbedded) ++ testDependencies
595597
override lazy val masterDependencies: Seq[sbt.ModuleID] = testDependencies ++ allAkka
596598
override lazy val producersDependencies: Seq[sbt.ModuleID] = testDependencies ++ allAkka
597599
override lazy val consumersSparkDependencies
598-
: Seq[sbt.ModuleID] = Seq(quartz, nameOf) ++ wireMock ++ hbase ++ avro4sTestAndDarwin ++ testDependencies ++ allAkka ++ avro4sTestAndDarwin
599-
override lazy val pluginElasticSparkDependencies
600+
: Seq[sbt.ModuleID] = Seq(quartz, nameOf) ++ wireMock ++ hbase ++ avro4sTestAndDarwin ++ testDependencies ++ allAkka ++ avro4sTestAndDarwin
601+
override lazy val pluginElasticSparkDependencies
600602
: Seq[sbt.ModuleID] = Seq(elasticSearch, elasticSearchSpark) ++ testDependencies
601603
override lazy val pluginHbaseSparkDependencies: Seq[sbt.ModuleID] = testDependencies
602604
override lazy val pluginPlainHbaseWriterSparkDependencies: Seq[sbt.ModuleID] = hbase ++ testDependencies ++ Seq(
@@ -653,8 +655,8 @@ class CDP717Dependencies(versions: CDP717Versions) extends Dependencies {
653655
Seq("com.github.tomakehurst" % "wiremock-standalone" % "2.25.0" % Test, "xmlunit" % "xmlunit" % "1.6" % Test)
654656

655657
override val awsAuth: Seq[ModuleID] = Seq(
656-
"org.apache.hadoop" % "hadoop-aws" % versions.hadoop,
657-
"org.apache.hadoop" % "hadoop-common" % versions.hadoop,
658-
"com.amazonaws"%"aws-java-sdk-bundle" % versions.awsBundle force()
658+
"org.apache.hadoop" % "hadoop-aws" % versions.hadoop,
659+
"org.apache.hadoop" % "hadoop-common" % versions.hadoop,
660+
"com.amazonaws" % "aws-java-sdk-bundle" % versions.awsBundle force ()
659661
)
660662
}

‎project/Cdh6Dependencies.scala‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ class Cdh6Dependencies(versions: Cdh6Versions) extends Dependencies {
1616
"com.fasterxml.jackson.module" % "jackson-module-scala_2.11" % "2.10.1" force ()
1717
)
1818

19-
lazy val mongoTest = "com.github.simplyscala" %% "scalatest-embedmongo" % "0.2.4" % Test
19+
lazy val mongoTest = "de.flapdoodle.embed" % "de.flapdoodle.embed.mongo" % "3.4.6" % Test
2020

2121
implicit class Exclude(module: ModuleID) {
2222
def log4jExclude: ModuleID =
@@ -244,8 +244,8 @@ class Cdh6Dependencies(versions: Cdh6Versions) extends Dependencies {
244244

245245
val avro4s = Seq(avro4sCore, avro4sJson)
246246

247-
val avro4sTest = avro4s.map(_ % Test)
248-
val avro4sTestAndDarwin = avro4sTest ++ Seq(darwinMockConnector % Test)
247+
val avro4sTest = avro4s.map(_ % Test)
248+
val avro4sTestAndDarwin = avro4sTest ++ Seq(darwinMockConnector % Test)
249249

250250
val jaxRs = "jakarta.ws.rs" % "jakarta.ws.rs-api" % "2.1.5"
251251

@@ -403,7 +403,7 @@ class Cdh6Dependencies(versions: Cdh6Versions) extends Dependencies {
403403

404404
val pluginMongoSparkDependencies = Seq(
405405
("org.mongodb.spark" %% "mongo-spark-connector" % "2.4.3").exclude("org.mongodb", "mongo-java-driver"),
406-
"org.mongodb" % "mongo-java-driver" % "3.12.2"
406+
"org.mongodb" % "mongo-java-driver" % "3.12.2"
407407
).map(excludeNetty)
408408

409409
val pluginMailerSparkDependencies = Seq(
@@ -474,9 +474,10 @@ class Cdh6Dependencies(versions: Cdh6Versions) extends Dependencies {
474474
val repositoryCoreDependencies: Seq[ModuleID] = testDependencies ++ Seq(shapeless)
475475
override val sparkPluginBasicDependencies: Seq[ModuleID] = scalaTestDependencies
476476
override val awsAuth: Seq[ModuleID] = Seq(
477-
"org.apache.hadoop" % "hadoop-aws" % versions.hadoop,
478-
"org.apache.hadoop" % "hadoop-common" % versions.hadoop,
479-
"com.amazonaws"%"aws-java-sdk-bundle" % versions.awsBundle force())
477+
"org.apache.hadoop" % "hadoop-aws" % versions.hadoop,
478+
"org.apache.hadoop" % "hadoop-common" % versions.hadoop,
479+
"com.amazonaws" % "aws-java-sdk-bundle" % versions.awsBundle force ()
480+
)
480481

481482
override val whitelabelMasterScriptClasspath = scriptClasspath += ""
482483
override val whitelabelProducerScriptClasspath = scriptClasspath += ""

‎project/EMR212Dependencies.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,8 @@ class EMR212Dependencies(val versions: EMR212Versions)
5757
lazy val joptSimpleTests = "net.sf.jopt-simple" % "jopt-simple" % versions.jopt % Test
5858
lazy val jettySecurity = "org.eclipse.jetty" % "jetty-security" % versions.jettySecurity
5959
lazy val avro4sTestAndDarwin = avro4sTest ++ Seq(darwinMockConnector % Test)
60-
lazy val mongoTest = "com.github.simplyscala" %% "scalatest-embedmongo" % "0.2.4" % Test
61-
lazy val shapeless = "com.chuusai" %% "shapeless" % "2.3.3"
60+
lazy val mongoTest = "de.flapdoodle.embed" % "de.flapdoodle.embed.mongo" % "3.4.6" % Test
61+
lazy val shapeless = "com.chuusai" %% "shapeless" % "2.3.3"
6262

6363
val jacksonTestDependencies = Seq(
6464
"com.fasterxml.jackson.core" % "jackson-annotations" % "2.10.1" % Test force (),

‎project/Vanilla2Dependencies.scala‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,8 @@ class Vanilla2Dependencies(val versions: Vanilla2Versions)
4040
lazy val joptSimpleTests = "net.sf.jopt-simple" % "jopt-simple" % versions.jopt % Test
4141
lazy val jettySecurity = "org.eclipse.jetty" % "jetty-security" % versions.jettySecurity
4242
lazy val avro4sTestAndDarwin = avro4sTest ++ Seq(darwinMockConnector % Test)
43-
lazy val mongoTest = "com.github.simplyscala" %% "scalatest-embedmongo" % "0.2.4" % Test
44-
lazy val shapeless = "com.chuusai" %% "shapeless" % "2.3.3"
43+
lazy val mongoTest = "de.flapdoodle.embed" % "de.flapdoodle.embed.mongo" % "3.4.6" % Test
44+
lazy val shapeless = "com.chuusai" %% "shapeless" % "2.3.3"
4545

4646
val jacksonTestDependencies = Seq(
4747
"com.fasterxml.jackson.core" % "jackson-annotations" % "2.10.1" % Test force (),
@@ -212,9 +212,10 @@ class Vanilla2Dependencies(val versions: Vanilla2Versions)
212212
)
213213

214214
override val awsAuth: Seq[ModuleID] = Seq(
215-
"org.apache.hadoop" % "hadoop-aws" % versions.hadoop,
216-
"org.apache.hadoop" % "hadoop-common" % versions.hadoop,
217-
"com.amazonaws"%"aws-java-sdk-bundle" % versions.awsBundle force())
215+
"org.apache.hadoop" % "hadoop-aws" % versions.hadoop,
216+
"org.apache.hadoop" % "hadoop-common" % versions.hadoop,
217+
"com.amazonaws" % "aws-java-sdk-bundle" % versions.awsBundle force ()
218+
)
218219

219220
override val pluginParallelWriteSparkDependencies: Seq[ModuleID] =
220221
Seq(scalaTest) ++ pluginHttpSparkDependencies ++ Seq(delta, hadoopAWS)

‎project/Vanilla2Versions.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ class Vanilla2Versions {
3535
val scalaParserAndCombinators = "1.0.4"
3636
val nifi = "1.11.4"
3737
val dbcp2Version = "2.4.0"
38-
val postgresqlEmbeddedVersion = "0.13.1"
38+
val postgresqlEmbeddedVersion = "0.13.4"
3939
val sttpVersion = "2.1.2"
4040
val reflectionsVersion = "0.9.11"
4141
val postgresqlVersion = "42.2.5"

‎repository/mongo/src/test/scala/com/github/simplyscala/MongoEmbedDatabase.scala‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package com.github.simplyscala
2+
3+
import de.flapdoodle.embed.mongo.distribution.Version
4+
import de.flapdoodle.embed.mongo.{MongodExecutable, MongodProcess, MongodStarter}
5+
import de.flapdoodle.embed.mongo.config.{Defaults, MongodConfig, Net}
6+
import de.flapdoodle.embed.mongo.packageresolver.Command
7+
import de.flapdoodle.embed.process.runtime.Network
8+
import de.flapdoodle.embed.process.config.RuntimeConfig
9+
import de.flapdoodle.embed.process.config.process.ProcessOutput
10+
11+
12+
/**
13+
* Extends this trait provide to your test class a connection to embedMongo database
14+
*/
15+
trait MongoEmbedDatabase {
16+
17+
private val runtimeConfig = Defaults.runtimeConfigFor(Command.MongoD)
18+
.processOutput(ProcessOutput.silent())
19+
.build()
20+
21+
protected def mongoStart(port: Int = 12345,
22+
host: String = "127.0.0.1",
23+
version: Version = Version.V3_6_22,
24+
runtimeConfig: RuntimeConfig = runtimeConfig): MongodProps = {
25+
val mongodExe: MongodExecutable = mongodExec(host, port, version, runtimeConfig)
26+
MongodProps(mongodExe.start(), mongodExe)
27+
}
28+
29+
protected def mongoStop( mongodProps: MongodProps ) = {
30+
Option(mongodProps).foreach( _.mongodProcess.stop() )
31+
Option(mongodProps).foreach( _.mongodExe.stop() )
32+
}
33+
34+
protected def withEmbedMongoFixture(port: Int = 12345,
35+
host: String = "127.0.0.1",
36+
version: Version = Version.V3_6_22,
37+
runtimeConfig: RuntimeConfig = runtimeConfig)
38+
(fixture: MongodProps => Any) {
39+
val mongodProps = mongoStart(port, host, version, runtimeConfig)
40+
try { fixture(mongodProps) } finally { Option(mongodProps).foreach( mongoStop ) }
41+
}
42+
43+
private def runtime(config: RuntimeConfig): MongodStarter = MongodStarter.getInstance(config)
44+
45+
private def mongodExec(host: String, port: Int, version: Version, runtimeConfig: RuntimeConfig): MongodExecutable =
46+
runtime(runtimeConfig).prepare(
47+
MongodConfig.builder()
48+
.version(version)
49+
.net(new Net(host, port, Network.localhostIsIPv6()))
50+
.build()
51+
)
52+
}
53+
54+
sealed case class MongodProps(mongodProcess: MongodProcess, mongodExe: MongodExecutable)

‎repository/postgres/src/test/scala/it/agilelab/bigdata/wasp/repository/postgres/AllPostgresTests.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,5 @@ class AllPostgresTests
2525
with SqlSourceBlImplTest
2626
with TopicBLImplTest
2727
with WebsocketBLImplTest
28-
with packageTest
28+
with packageTest {
29+
}

‎repository/postgres/src/test/scala/it/agilelab/bigdata/wasp/repository/postgres/bl/BatchJobBLImplTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ trait BatchJobBLImplTest {
77

88
self : PostgresSuite =>
99

10-
val batchJobBL = BatchJobBLImpl(pgDB)
10+
lazy val batchJobBL = BatchJobBLImpl(pgDB)
1111

1212

1313
it should "test batchJobBL" in {

‎repository/postgres/src/test/scala/it/agilelab/bigdata/wasp/repository/postgres/bl/BatchJobInstanceBLImplTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import it.agilelab.bigdata.wasp.repository.postgres.utils.PostgresSuite
77
trait BatchJobInstanceBLImplTest {
88
self : PostgresSuite =>
99

10-
private val bl = BatchJobInstanceBLImpl(pgDB)
10+
lazy private val bl = BatchJobInstanceBLImpl(pgDB)
1111

1212

1313
it should "test batchJobInstanceBL" in {

‎repository/postgres/src/test/scala/it/agilelab/bigdata/wasp/repository/postgres/bl/BatchSchedulersBLImplTest.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import it.agilelab.bigdata.wasp.repository.postgres.utils.PostgresSuite
66
trait BatchSchedulersBLImplTest {
77

88
self: PostgresSuite =>
9-
private val bl = BatchSchedulersBLImpl(pgDB)
9+
private lazy val bl = BatchSchedulersBLImpl(pgDB)
1010

1111
it should "Test BatchSchedulersBLImpl" in {
1212

0 commit comments

Comments
 (0)