[RORDEV-1567] ECS serializer and improved configurable serializer

audit/src/main/scala/tech/beshu/ror/audit/utils/AuditSerializationHelper.scala

@@ -100,6 +100,7 @@ private[ror] object AuditSerializationHelper {
       case AuditFieldValueDescriptor.InvolvedIndices => if (requestContext.involvesIndices) requestContext.indices.toList.asJava else List.empty.asJava
       case AuditFieldValueDescriptor.AclHistory => requestContext.history
       case AuditFieldValueDescriptor.ProcessingDurationMillis => eventData.duration.toMillis
+      case AuditFieldValueDescriptor.ProcessingDurationNanos => eventData.duration.toNanos
       case AuditFieldValueDescriptor.Timestamp => timestampFormatter.format(requestContext.timestamp)
       case AuditFieldValueDescriptor.Id => requestContext.id
       case AuditFieldValueDescriptor.CorrelationId => requestContext.correlationId
@@ -119,7 +120,17 @@ private[ror] object AuditSerializationHelper {
       case AuditFieldValueDescriptor.EsNodeName => eventData.requestContext.auditEnvironmentContext.esNodeName
       case AuditFieldValueDescriptor.EsClusterName => eventData.requestContext.auditEnvironmentContext.esClusterName
       case AuditFieldValueDescriptor.StaticText(text) => text
+      case AuditFieldValueDescriptor.NumericValue(value) => value
+      case AuditFieldValueDescriptor.BooleanValue(value) => value
       case AuditFieldValueDescriptor.Combined(values) => values.map(resolver(eventData)).mkString
+      case AuditFieldValueDescriptor.Nested(values) =>
+        val resolveAuditFieldValue = resolver(eventData)
+        val nestedFields: Map[String, Any] = values.map { case (nestedName, nestedDescriptor) =>
+          nestedName.value -> resolveAuditFieldValue(nestedDescriptor)
+        }
+        nestedFields.foldLeft(new JSONObject()) { case (soFar, (key, value)) =>
+          soFar.put(key, value)
+        }
     }
   }
 
@@ -178,6 +189,8 @@ private[ror] object AuditSerializationHelper {
 
     case object ProcessingDurationMillis extends AuditFieldValueDescriptor
 
+    case object ProcessingDurationNanos extends AuditFieldValueDescriptor
+
     // Identifiers
     case object Timestamp extends AuditFieldValueDescriptor
 
@@ -223,8 +236,18 @@ private[ror] object AuditSerializationHelper {
 
     final case class StaticText(value: String) extends AuditFieldValueDescriptor
 
+    final case class BooleanValue(value: Boolean) extends AuditFieldValueDescriptor
+
+    final case class NumericValue(value: Double) extends AuditFieldValueDescriptor
+
     final case class Combined(values: List[AuditFieldValueDescriptor]) extends AuditFieldValueDescriptor
 
+    final case class Nested(values: Map[AuditFieldName, AuditFieldValueDescriptor]) extends AuditFieldValueDescriptor
+
+    object Nested {
+      def apply(elems: (AuditFieldName, AuditFieldValueDescriptor)*) = new Nested(elems.toMap)
+    }
+
   }
 
   sealed trait AuditFieldGroup

core/src/main/scala/tech/beshu/ror/accesscontrol/audit/configurable/AuditFieldValueDescriptorParser.scala

@@ -65,6 +65,7 @@ object AuditFieldValueDescriptorParser {
       case "INVOLVED_INDICES" => Some(AuditFieldValueDescriptor.InvolvedIndices)
       case "ACL_HISTORY" => Some(AuditFieldValueDescriptor.AclHistory)
       case "PROCESSING_DURATION_MILLIS" => Some(AuditFieldValueDescriptor.ProcessingDurationMillis)
+      case "PROCESSING_DURATION_NANOS" => Some(AuditFieldValueDescriptor.ProcessingDurationNanos)
       case "TIMESTAMP" => Some(AuditFieldValueDescriptor.Timestamp)
       case "ID" => Some(AuditFieldValueDescriptor.Id)
       case "CORRELATION_ID" => Some(AuditFieldValueDescriptor.CorrelationId)

core/src/main/scala/tech/beshu/ror/accesscontrol/audit/ecs/EcsV1AuditLogSerializer.scala

@@ -0,0 +1,87 @@
+/*
+ *    This file is part of ReadonlyREST.
+ *
+ *    ReadonlyREST is free software: you can redistribute it and/or modify
+ *    it under the terms of the GNU General Public License as published by
+ *    the Free Software Foundation, either version 3 of the License, or
+ *    (at your option) any later version.
+ *
+ *    ReadonlyREST is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU General Public License for more details.
+ *
+ *    You should have received a copy of the GNU General Public License
+ *    along with ReadonlyREST.  If not, see http://www.gnu.org/licenses/
+ */
+package tech.beshu.ror.accesscontrol.audit.ecs
+
+import org.json.JSONObject
+import tech.beshu.ror.accesscontrol.audit.ecs.EcsV1AuditLogSerializer.fields
+import tech.beshu.ror.audit.utils.AuditSerializationHelper
+import tech.beshu.ror.audit.utils.AuditSerializationHelper.{AllowedEventMode, AuditFieldName, AuditFieldValueDescriptor}
+import tech.beshu.ror.audit.{AuditLogSerializer, AuditResponseContext}
+
+class EcsV1AuditLogSerializer(val allowedEventMode: AllowedEventMode) extends AuditLogSerializer {
+
+  override def onResponse(responseContext: AuditResponseContext): Option[JSONObject] = {
+    AuditSerializationHelper.serialize(responseContext, fields, allowedEventMode)
+  }
+
+}
+
+object EcsV1AuditLogSerializer {
+  private val fields: Map[AuditFieldName, AuditFieldValueDescriptor] = Map(
+    AuditFieldName("ecs") -> AuditFieldValueDescriptor.Nested(
+      // Schema defined by EcsV1AuditLogSerializer is ECS 1.4.0 compliant and does not use newer features introduced by later versions
+      AuditFieldName("version") -> AuditFieldValueDescriptor.StaticText("1.4.0"),
+    ),
+    AuditFieldName("@timestamp") -> AuditFieldValueDescriptor.Timestamp,
+    AuditFieldName("trace") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("id") -> AuditFieldValueDescriptor.CorrelationId,
+    ),
+    AuditFieldName("url") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("path") -> AuditFieldValueDescriptor.HttpPath,
+    ),
+    AuditFieldName("source") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("address") -> AuditFieldValueDescriptor.RemoteAddress,
+    ),
+    AuditFieldName("destination") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("address") -> AuditFieldValueDescriptor.LocalAddress,
+    ),
+    AuditFieldName("http") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("request") -> AuditFieldValueDescriptor.Nested(
+        AuditFieldName("method") -> AuditFieldValueDescriptor.HttpMethod,
+        AuditFieldName("body") -> AuditFieldValueDescriptor.Nested(
+          AuditFieldName("content") -> AuditFieldValueDescriptor.Content,
+          AuditFieldName("bytes") -> AuditFieldValueDescriptor.ContentLengthInBytes,
+        ),
+      ),
+    ),
+    AuditFieldName("user") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("name") -> AuditFieldValueDescriptor.User,
+      AuditFieldName("effective") -> AuditFieldValueDescriptor.Nested(
+        AuditFieldName("name") -> AuditFieldValueDescriptor.ImpersonatedByUser,
+      ),
+    ),
+    AuditFieldName("event") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("id") -> AuditFieldValueDescriptor.Id,
+      AuditFieldName("action") -> AuditFieldValueDescriptor.Action,
+      AuditFieldName("type") -> AuditFieldValueDescriptor.Type,
+      AuditFieldName("reason") -> AuditFieldValueDescriptor.FinalState,
+      AuditFieldName("duration") -> AuditFieldValueDescriptor.ProcessingDurationNanos,
+    ),
+    AuditFieldName("error") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("type") -> AuditFieldValueDescriptor.ErrorType,
+      AuditFieldName("message") -> AuditFieldValueDescriptor.ErrorMessage,
+    ),
+    AuditFieldName("labels") -> AuditFieldValueDescriptor.Nested(
+      AuditFieldName("es_cluster_name") -> AuditFieldValueDescriptor.EsClusterName,
+      AuditFieldName("es_node_name") -> AuditFieldValueDescriptor.EsNodeName,
+      AuditFieldName("es_task_id") -> AuditFieldValueDescriptor.TaskId,
+      AuditFieldName("involved_indices") -> AuditFieldValueDescriptor.InvolvedIndices,
+      AuditFieldName("acl_history") -> AuditFieldValueDescriptor.AclHistory,
+      AuditFieldName("x_forwarded_for") -> AuditFieldValueDescriptor.XForwardedForHttpHeader,
+    ),
+  )
+}

core/src/main/scala/tech/beshu/ror/accesscontrol/factory/decoders/AuditingSettingsDecoder.scala

@@ -18,25 +18,26 @@ package tech.beshu.ror.accesscontrol.factory.decoders
 
 import cats.data.NonEmptyList
 import io.circe.Decoder.*
-import io.circe.{Decoder, DecodingFailure, Json, HCursor, KeyDecoder}
+import io.circe.*
 import io.lemonlabs.uri.Uri
 import org.apache.logging.log4j.scala.Logging
 import tech.beshu.ror.accesscontrol.audit.AuditingTool
 import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink
 import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink.Config
 import tech.beshu.ror.accesscontrol.audit.AuditingTool.AuditSettings.AuditSink.Config.{EsDataStreamBasedSink, EsIndexBasedSink, LogBasedSink}
 import tech.beshu.ror.accesscontrol.audit.configurable.{AuditFieldValueDescriptorParser, ConfigurableAuditLogSerializer}
+import tech.beshu.ror.accesscontrol.audit.ecs.EcsV1AuditLogSerializer
 import tech.beshu.ror.accesscontrol.domain.RorAuditIndexTemplate.CreationError
 import tech.beshu.ror.accesscontrol.domain.{AuditCluster, RorAuditDataStream, RorAuditIndexTemplate, RorAuditLoggerName}
 import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCreationError.Reason.Message
 import tech.beshu.ror.accesscontrol.factory.RawRorConfigBasedCoreFactory.CoreCreationError.{AuditingSettingsCreationError, Reason}
 import tech.beshu.ror.accesscontrol.factory.decoders.common.{lemonLabsUriDecoder, nonEmptyStringDecoder}
 import tech.beshu.ror.accesscontrol.utils.CirceOps.{AclCreationErrorCoders, DecodingFailureOps}
 import tech.beshu.ror.accesscontrol.utils.SyncDecoderCreator
+import tech.beshu.ror.audit.AuditLogSerializer
 import tech.beshu.ror.audit.AuditResponseContext.Verbosity
-import tech.beshu.ror.audit.utils.AuditSerializationHelper.{AllowedEventMode, AuditFieldName, AuditFieldValueDescriptor}
 import tech.beshu.ror.audit.adapters.*
-import tech.beshu.ror.audit.AuditLogSerializer
+import tech.beshu.ror.audit.utils.AuditSerializationHelper.{AllowedEventMode, AuditFieldName, AuditFieldValueDescriptor}
 import tech.beshu.ror.es.EsVersion
 import tech.beshu.ror.implicits.*
 import tech.beshu.ror.utils.yaml.YamlKeyDecoder
@@ -217,10 +218,26 @@ object AuditingSettingsDecoder extends Logging {
           c.downField("serializer").as[Option[AuditLogSerializer]](extendedSyntaxStaticSerializerDecoder)
         case SerializerType.ExtendedSyntaxConfigurableSerializer =>
           c.downField("serializer").as[Option[AuditLogSerializer]](extendedSyntaxConfigurableSerializerDecoder)
+        case SerializerType.EcsSerializer =>
+          c.downField("serializer").as[Option[AuditLogSerializer]](ecsSerializerDecoder)
       }
     } yield result
   }
 
+  private def ecsSerializerDecoder: Decoder[Option[AuditLogSerializer]] = Decoder.instance { c =>
+    for {
+      version <- c.downField("version").as[Option[EcsSerializerVersion]]
+      allowedEventMode <- c.downField("verbosity_level_serialization_mode").as[AllowedEventMode]
+        .left.map(withAuditingSettingsCreationErrorMessage(msg => s"ECS serializer is used, but the 'verbosity_level_serialization_mode' setting is invalid: $msg"))
+      serializer = version match {
+        case None =>
+          new EcsV1AuditLogSerializer(allowedEventMode)
+        case Some(EcsSerializerVersion.V1) =>
+          new EcsV1AuditLogSerializer(allowedEventMode)
+      }
+    } yield Some(serializer)
+  }
+
   private def extendedSyntaxConfigurableSerializerDecoder: Decoder[Option[AuditLogSerializer]] = Decoder.instance { c =>
     for {
       allowedEventMode <- c.downField("verbosity_level_serialization_mode").as[AllowedEventMode]
@@ -265,9 +282,11 @@ object AuditingSettingsDecoder extends Logging {
             Right(SerializerType.ExtendedSyntaxStaticSerializer)
           case "configurable" =>
             Right(SerializerType.ExtendedSyntaxConfigurableSerializer)
+          case "ecs" =>
+            Right(SerializerType.EcsSerializer)
           case other =>
             Left(DecodingFailure(AclCreationErrorCoders.stringify(
-              AuditingSettingsCreationError(Message(s"Invalid serializer type '$other', allowed values [static, configurable]"))
+              AuditingSettingsCreationError(Message(s"Invalid serializer type '$other', allowed values [static, configurable, ecs]"))
             ), Nil))
         }
       case Some(_) | None =>
@@ -283,6 +302,19 @@ object AuditingSettingsDecoder extends Logging {
     case object ExtendedSyntaxStaticSerializer extends SerializerType
 
     case object ExtendedSyntaxConfigurableSerializer extends SerializerType
+
+    case object EcsSerializer extends SerializerType
+  }
+
+  private given ccsSerializerVersionDecoder: Decoder[EcsSerializerVersion] = Decoder.decodeString.map(_.toLowerCase).emap {
+    case "v1" => Right(EcsSerializerVersion.V1)
+    case other => Left(s"Invalid ECS serializer version $other")
+  }
+
+  private sealed trait EcsSerializerVersion
+
+  private object EcsSerializerVersion {
+    case object V1 extends EcsSerializerVersion
   }
 
   private def withAuditingSettingsCreationErrorMessage(message: String => String)(decodingFailure: DecodingFailure) = {
@@ -341,11 +373,23 @@ object AuditingSettingsDecoder extends Logging {
     KeyDecoder.decodeKeyString.map(AuditFieldName.apply)
   }
 
-  given auditFieldValueDecoder: Decoder[AuditFieldValueDescriptor] = {
-    SyncDecoderCreator
-      .from(Decoder.decodeString)
-      .emap(AuditFieldValueDescriptorParser.parse)
-      .decoder
+  given auditFieldValueDecoder: Decoder[AuditFieldValueDescriptor] = Decoder.instance { cursor =>
+    cursor.value.fold(
+      jsonNull = Left(DecodingFailure("Expected AuditFieldValueDescriptor, got null", cursor.history)),
+      jsonBoolean = b => Right(AuditFieldValueDescriptor.BooleanValue(b)),
+      jsonNumber = n => Right(AuditFieldValueDescriptor.NumericValue(n.toDouble)),
+      jsonString = s => AuditFieldValueDescriptorParser.parse(s).left.map(err => DecodingFailure(err, cursor.history)),
+      jsonArray = _ => Left(DecodingFailure("AuditFieldValueDescriptor cannot be an array", cursor.history)),
+      jsonObject = obj => {
+        val decoded = obj.toMap.toList.traverse { case (k, v) =>
+          for {
+            name <- Right(AuditFieldName(k))
+            value <- v.as[AuditFieldValueDescriptor](using auditFieldValueDecoder)
+          } yield name -> value
+        }
+        decoded.map(pairs => AuditFieldValueDescriptor.Nested(pairs.toMap))
+      }
+    )
   }
 
   given verbosityDecoder: Decoder[Verbosity] = {