-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
DP-2918: continue on data error (#62)
* DP-2918: drop or serialise collections in schema inference as well * DP-2918: add a test for dropping arrays in normalization * DP-2918: Return Either in normalizer, and fail in the task when normalization fails * DP-2918: Put failing records in DLQ * DP-2918: Change exception name * DP-2918: Introduce ErrorPolicyConfig class * DP-2918: Tests for ErrorPolicyConfig * DP-2918: Add continueOninvalidInput configuration * DP-2918: Call errand record reporter only for InvalidInputException * DP-2918: Change names of a test * DP-2918: Scalafmt * DP-2918: Fix maxretries reset in case of continue on error * DP-2918: Add comment * DP-2918: InvalidInputErrorHandler test * DP-2918: Fix InvalidInputErrorHandler * DP-2918: Integrate InvalidInputErrorHandler into EmsSinkTask * DP-2918: Add a comment * DP-2918: Add e2e test * DP-2918: Remove a comment * DP-2918: Scalafmt * DP-2918: remove a println
- Loading branch information
Showing
24 changed files
with
562 additions
and
224 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
75 changes: 75 additions & 0 deletions
75
connector/src/main/scala/com/celonis/kafka/connect/ems/config/ErrorPolicyConfig.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| /* | ||
| * Copyright 2024 Celonis SE | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.celonis.kafka.connect.ems.config | ||
|
|
||
| import cats.syntax.either._ | ||
| import com.celonis.kafka.connect.ems.config.EmsSinkConfigConstants.ERROR_CONTINUE_ON_INVALID_INPUT_DEFAULT | ||
| import com.celonis.kafka.connect.ems.config.EmsSinkConfigConstants.ERROR_CONTINUE_ON_INVALID_INPUT_KEY | ||
| import com.celonis.kafka.connect.ems.config.EmsSinkConfigConstants.ERROR_POLICY_DOC | ||
| import com.celonis.kafka.connect.ems.config.EmsSinkConfigConstants.ERROR_POLICY_KEY | ||
| import com.celonis.kafka.connect.ems.config.ErrorPolicyConfig.ErrorPolicyType | ||
| import com.celonis.kafka.connect.ems.config.ErrorPolicyConfig.ErrorPolicyType.CONTINUE | ||
| import com.celonis.kafka.connect.ems.config.ErrorPolicyConfig.ErrorPolicyType.RETRY | ||
| import com.celonis.kafka.connect.ems.config.ErrorPolicyConfig.ErrorPolicyType.THROW | ||
| import com.celonis.kafka.connect.ems.config.PropertiesHelper.error | ||
| import com.celonis.kafka.connect.ems.config.PropertiesHelper.getBoolean | ||
| import com.celonis.kafka.connect.ems.config.PropertiesHelper.nonEmptyStringOr | ||
| import com.celonis.kafka.connect.ems.errors.ErrorPolicy | ||
| import com.celonis.kafka.connect.ems.errors.InvalidInputErrorHandler | ||
| import org.apache.kafka.connect.sink.ErrantRecordReporter | ||
|
|
||
| final case class ErrorPolicyConfig( | ||
| policyType: ErrorPolicyType, | ||
| retryConfig: RetryConfig, | ||
| continueOnInvalidInput: Boolean, | ||
| ) { | ||
| lazy val errorPolicy: ErrorPolicy = | ||
| policyType match { | ||
| case ErrorPolicyType.THROW => ErrorPolicy.Throw | ||
| case ErrorPolicyType.CONTINUE => ErrorPolicy.Continue | ||
| case ErrorPolicyType.RETRY => ErrorPolicy.Retry | ||
| } | ||
|
|
||
| def invalidInputErrorHandler(reporter: Option[ErrantRecordReporter]): InvalidInputErrorHandler = | ||
| new InvalidInputErrorHandler(continueOnInvalidInput, reporter) | ||
| } | ||
|
|
||
| object ErrorPolicyConfig { | ||
| sealed trait ErrorPolicyType | ||
| object ErrorPolicyType { | ||
| case object THROW extends ErrorPolicyType | ||
| case object CONTINUE extends ErrorPolicyType | ||
| case object RETRY extends ErrorPolicyType | ||
| } | ||
|
|
||
| def extract(props: Map[String, _]): Either[String, ErrorPolicyConfig] = | ||
| for { | ||
| policyType <- extractType(props) | ||
| retryConfig <- RetryConfig.extractRetry(props) | ||
| continueOnInvalidInput = | ||
| getBoolean(props, ERROR_CONTINUE_ON_INVALID_INPUT_KEY).getOrElse(ERROR_CONTINUE_ON_INVALID_INPUT_DEFAULT) | ||
| } yield ErrorPolicyConfig(policyType, retryConfig, continueOnInvalidInput = continueOnInvalidInput) | ||
|
|
||
| private def extractType(props: Map[String, _]): Either[String, ErrorPolicyType] = | ||
| nonEmptyStringOr(props, ERROR_POLICY_KEY, ERROR_POLICY_DOC).map(_.toUpperCase) | ||
| .flatMap { | ||
| case "THROW" => THROW.asRight | ||
| case "RETRY" => RETRY.asRight | ||
| case "CONTINUE" => CONTINUE.asRight | ||
| case _ => error(ERROR_POLICY_KEY, ERROR_POLICY_DOC) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
connector/src/main/scala/com/celonis/kafka/connect/ems/errors/InvalidInputErrorHandler.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| /* | ||
| * Copyright 2024 Celonis SE | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.celonis.kafka.connect.ems.errors | ||
|
|
||
| import com.typesafe.scalalogging.StrictLogging | ||
| import org.apache.kafka.connect.sink.ErrantRecordReporter | ||
| import org.apache.kafka.connect.sink.SinkRecord | ||
|
|
||
| /** Error policies work at the batch level, while this handler works at the record level. It works only for | ||
| * InvalidInputExceptions, that are errors due to defects of single records. | ||
| */ | ||
| final class InvalidInputErrorHandler( | ||
| continueOnInvalidInput: Boolean, | ||
| errantRecordReporter: Option[ErrantRecordReporter], | ||
| ) extends StrictLogging { | ||
| def handle(record: SinkRecord, error: Throwable): Unit = error match { | ||
| case _: InvalidInputException if continueOnInvalidInput => | ||
| logger.warn("Error policy is set to CONTINUE on InvalidInput", error) | ||
| errantRecordReporter.foreach(errantRecordReporter => errantRecordReporter.report(record, error)) | ||
| case _ => throw error | ||
| } | ||
| } |
Oops, something went wrong.