[#550] Parallel Write Plugin - improving error handling when sending streamed data

# New features and improvements

- Parallel Write writer throws exception if the http response status is not 2xx, therefore now spark queries will fail if the response of data/stream is not successful.
- Add more logs around the parallel write http calls, to see them, enable logger category `it.agilelab.bigdata.wasp.consumers.spark.plugins.parallel` at INFO level
- Avoid performing parallel write when there are no partitions to be written (streaming query first "run")
- Update image used to build `waspctl`
- Automatically generate openapi during the build instead of using the committed one
- Update `de.flapdoodle.embed:de.flapdoodle.embed.mongo` from `3.4.6` to `3.5.4`
- Change restlet repo from `https://maven.restlet.com/` to `https://maven.restlet.talend.com` since the SSL certificate was expired

# Breaking changes

None.

# Migration

None.

# Bug fixes

None.

# How this feature was tested

Existing unit tests.

# Related issue

Closes #550

Author: Pietro Brunetti

.gitlab-ci.yml

@@ -30,7 +30,7 @@ test-kernel:
   image: "hseeberger/scala-sbt:8u312_1.6.2_2.12.15"
   parallel:
     matrix:
-      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
+      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
   script:
     - ./run-sbt-unprivileged.sh "clean coverageOn wasp-kernel/test coverageOff wasp-kernel/coverageReport wasp-kernel/coverageAggregate"
   after_script:
@@ -56,7 +56,7 @@ test-plugin:
   image: "hseeberger/scala-sbt:8u312_1.6.2_2.12.15"
   parallel:
     matrix:
-      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
+      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
   script:
     - ./run-sbt-unprivileged.sh "clean coverageOn wasp-plugin/test coverageOff wasp-plugin/coverageReport wasp-plugin/coverageAggregate"
   after_script:
@@ -83,7 +83,7 @@ test-repo:
   image: "hseeberger/scala-sbt:8u312_1.6.2_2.12.15"
   parallel:
     matrix:
-      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
+      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
   script:
     - ./run-sbt-unprivileged.sh "clean coverageOn wasp-repository/test coverageOff wasp-repository/coverageReport wasp-repository/coverageAggregate"
   after_script:
@@ -109,7 +109,7 @@ compile-whitelabel:
   image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
   parallel:
     matrix:
-      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
+      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
   script:
     - ./run-sbt-unprivileged.sh "clean wasp-whitelabel/Test/compile"
   tags:
@@ -128,11 +128,27 @@ censor:
   script:
     - ./censor.sh
 
+generate-open-api:
+  stage: test
+  interruptible: true
+  image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
+  script:
+    - ./run-sbt-unprivileged.sh "generateOpenApi"
+  tags:
+    - gitlab-org
+  only:
+    - /^develop$/
+    - /(^release\/v([0-9]+)\.([0-9]+))/
+    - merge_requests
+  artifacts:
+    paths: ['documentation/wasp-openapi.yaml']
+  needs: []
+
 
 waspctl:
   interruptible: true
   stage: deploy
-  image: "registry.gitlab.com/agilefactory/agile.wasp2/waspctl-build:0.1.0"
+  image: "registry.gitlab.com/agilefactory/agile.wasp2/waspctl-build:0.2.0"
   script:
     - bash waspctl/build-for-all-archs.bash
   artifacts:
@@ -142,14 +158,18 @@ waspctl:
     - /(^release\/v([0-9]+)\.([0-9]+))/
     - merge_requests
     - tags
+  needs:
+    - generate-open-api
+  dependencies:
+    - generate-open-api
 
 snapshot:
   interruptible: false
   stage: deploy
   image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
   parallel:
     matrix:
-      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
+      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
   tags:
     - gitlab-org
   script:
@@ -173,7 +193,7 @@ release:
   image: "hseeberger/scala-sbt:11.0.14.1_1.6.2_2.12.15"
   parallel:
     matrix:
-      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "CDP717" , "EMR_2_12"]
+      - WASP_FLAVOR: [ "VANILLA2", "VANILLA2_2_12", "EMR_2_12", "CDP717"]
   script:
     - "mkdir -p $HOME/.sbt/gpg"
     - "echo $GPG_PUB | base64 -d > $HOME/.sbt/gpg/pubring.asc"

documentation/wasp-openapi.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.1
 info:
   title: wasp-api
-  version: 2.27.0-436-refactor-plugin-in-order-to-allow-plugin-defined-models-SNAPSHOT
+  version: 2.31.0-550-parallel-write-plugin-improving-error-handling-when-sending-streamed-data-SNAPSHOT
 servers:
 - url: http://localhost:2891
   description: default development server, beware of CORS
@@ -1114,9 +1114,7 @@ paths:
               - owner
               - isSystem
               - creationTime
-              - legacyStreamingComponents
               - structuredStreamingComponents
-              - rtComponents
               - labels
               - enrichmentSources
               type: object
@@ -1132,18 +1130,10 @@ paths:
                 creationTime:
                   type: integer
                   format: int64
-                legacyStreamingComponents:
-                  type: array
-                  items:
-                    $ref: '#/components/schemas/LegacyStreamingETLModel'
                 structuredStreamingComponents:
                   type: array
                   items:
                     $ref: '#/components/schemas/StructuredStreamingETLModel'
-                rtComponents:
-                  type: array
-                  items:
-                    $ref: '#/components/schemas/RTModel'
                 dashboard:
                   $ref: '#/components/schemas/DashboardModel'
                 labels:
@@ -1194,9 +1184,7 @@ paths:
               - owner
               - isSystem
               - creationTime
-              - legacyStreamingComponents
               - structuredStreamingComponents
-              - rtComponents
               - labels
               - enrichmentSources
               type: object
@@ -1212,18 +1200,10 @@ paths:
                 creationTime:
                   type: integer
                   format: int64
-                legacyStreamingComponents:
-                  type: array
-                  items:
-                    $ref: '#/components/schemas/LegacyStreamingETLModel'
                 structuredStreamingComponents:
                   type: array
                   items:
                     $ref: '#/components/schemas/StructuredStreamingETLModel'
-                rtComponents:
-                  type: array
-                  items:
-                    $ref: '#/components/schemas/RTModel'
                 dashboard:
                   $ref: '#/components/schemas/DashboardModel'
                 labels:
@@ -2620,40 +2600,6 @@ components:
       xml:
         name: CountEntry
         namespace: java://it.agilelab.bigdata.wasp.models
-    LegacyStreamingETLModel:
-      required:
-      - name
-      - inputs
-      - output
-      - mlModels
-      - kafkaAccessType
-      - group
-      - isActive
-      type: object
-      properties:
-        name:
-          type: string
-        inputs:
-          type: array
-          items:
-            $ref: '#/components/schemas/ReaderModel'
-        output:
-          $ref: '#/components/schemas/WriterModel'
-        mlModels:
-          type: array
-          items:
-            $ref: '#/components/schemas/MlModelOnlyInfo'
-        strategy:
-          $ref: '#/components/schemas/StrategyModel'
-        kafkaAccessType:
-          type: string
-        group:
-          type: string
-        isActive:
-          type: boolean
-      xml:
-        name: LegacyStreamingETLModel
-        namespace: java://it.agilelab.bigdata.wasp.models
     Sources:
       required:
       - found
@@ -2978,17 +2924,17 @@ components:
     GenericModel:
       required:
       - name
-      - kind
       - value
+      - product
       - options
       type: object
       properties:
         name:
           type: string
-        kind:
-          type: string
         value:
           $ref: '#/components/schemas/BsonDocument'
+        product:
+          $ref: '#/components/schemas/GenericProduct'
         options:
           $ref: '#/components/schemas/GenericOptions'
       xml:
@@ -3509,7 +3455,6 @@ components:
           type: object
           additionalProperties:
             type: string
-      nullable: true
       xml:
         name: WriterModel
         namespace: java://it.agilelab.bigdata.wasp.models
@@ -3520,9 +3465,7 @@ components:
       - owner
       - isSystem
       - creationTime
-      - legacyStreamingComponents
       - structuredStreamingComponents
-      - rtComponents
       - labels
       - enrichmentSources
       type: object
@@ -3538,18 +3481,10 @@ components:
         creationTime:
           type: integer
           format: int64
-        legacyStreamingComponents:
-          type: array
-          items:
-            $ref: '#/components/schemas/LegacyStreamingETLModel'
         structuredStreamingComponents:
           type: array
           items:
             $ref: '#/components/schemas/StructuredStreamingETLModel'
-        rtComponents:
-          type: array
-          items:
-            $ref: '#/components/schemas/RTModel'
         dashboard:
           $ref: '#/components/schemas/DashboardModel'
         labels:
@@ -3622,6 +3557,7 @@ components:
       - kryoSerializer
       - streamingBatchIntervalMs
       - checkpointDir
+      - enableHiveSupport
       - others
       - schedulingStrategy
       - name
@@ -3660,6 +3596,8 @@ components:
           format: int32
         checkpointDir:
           type: string
+        enableHiveSupport:
+          type: boolean
         triggerIntervalMs:
           type: integer
           format: int64
@@ -4190,28 +4128,6 @@ components:
       xml:
         name: SolrConfigModel
         namespace: java://it.agilelab.bigdata.wasp.models.configuration
-    RTModel:
-      required:
-      - name
-      - inputs
-      - isActive
-      type: object
-      properties:
-        name:
-          type: string
-        inputs:
-          type: array
-          items:
-            $ref: '#/components/schemas/ReaderModel'
-        isActive:
-          type: boolean
-        strategy:
-          $ref: '#/components/schemas/StrategyModel'
-        endpoint:
-          $ref: '#/components/schemas/WriterModel'
-      xml:
-        name: RTModel
-        namespace: java://it.agilelab.bigdata.wasp.models
     Events:
       required:
       - found
@@ -4465,6 +4381,19 @@ components:
       xml:
         name: KeyValueOption
         namespace: java://it.agilelab.bigdata.wasp.models
+    GenericProduct:
+      required:
+      - categoryName
+      type: object
+      properties:
+        categoryName:
+          type: string
+        productName:
+          type: string
+          nullable: true
+      xml:
+        name: GenericProduct
+        namespace: java://it.agilelab.bigdata.wasp.datastores
     PipegraphStatus:
       type: string
       enum: