diff --git a/backends-clickhouse/src-delta20/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider b/backends-clickhouse/src-delta20/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider deleted file mode 100644 index 2e9fc7f42f0..00000000000 --- a/backends-clickhouse/src-delta20/main/resources/META-INF/services/org.apache.gluten.sql.shims.DeltaShimProvider +++ /dev/null @@ -1 +0,0 @@ -org.apache.gluten.sql.shims.delta20.Delta20ShimProvider \ No newline at end of file diff --git a/backends-clickhouse/src-delta20/main/scala/io/delta/tables/ClickhouseTable.scala b/backends-clickhouse/src-delta20/main/scala/io/delta/tables/ClickhouseTable.scala deleted file mode 100644 index 98953fccd27..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/io/delta/tables/ClickhouseTable.scala +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.tables - -import org.apache.spark.sql.{Dataset, Row, SparkSession} -import org.apache.spark.sql.delta.{DeltaErrors, DeltaTableIdentifier, DeltaTableUtils} -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 - -import org.apache.hadoop.fs.Path - -class ClickhouseTable( - @transient private val _df: Dataset[Row], - @transient private val table: ClickHouseTableV2) - extends DeltaTable(_df, table) { - - override def optimize(): DeltaOptimizeBuilder = { - DeltaOptimizeBuilder( - sparkSession, - table.tableIdentifier.getOrElse(s"clickhouse.`${deltaLog.dataPath.toString}`")) - } -} - -object ClickhouseTable { - - /** - * Create a DeltaTable for the data at the given `path`. - * - * Note: This uses the active SparkSession in the current thread to read the table data. Hence, - * this throws error if active SparkSession has not been set, that is, - * `SparkSession.getActiveSession()` is empty. - * - * @since 0.3.0 - */ - def forPath(path: String): DeltaTable = { - val sparkSession = SparkSession.getActiveSession.getOrElse { - throw DeltaErrors.activeSparkSessionNotFound() - } - forPath(sparkSession, path) - } - - /** - * Create a DeltaTable for the data at the given `path` using the given SparkSession. - * - * @since 0.3.0 - */ - def forPath(sparkSession: SparkSession, path: String): DeltaTable = { - val hdpPath = new Path(path) - if (DeltaTableUtils.isDeltaTable(sparkSession, hdpPath)) { - new ClickhouseTable( - sparkSession.read.format("clickhouse").load(path), - new ClickHouseTableV2(spark = sparkSession, path = hdpPath) - ) - } else { - throw DeltaErrors.notADeltaTableException(DeltaTableIdentifier(path = Some(path))) - } - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala deleted file mode 100644 index 48d26498f12..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/parser/GlutenCacheFilesSqlParser.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.parser - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.parser.ParserInterface -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.types.{DataType, StructType} - -class GlutenCacheFilesSqlParser(spark: SparkSession, delegate: ParserInterface) - extends GlutenCacheFileSqlParserBase { - - override def parsePlan(sqlText: String): LogicalPlan = - parse(sqlText) { - parser => - astBuilder.visit(parser.singleStatement()) match { - case plan: LogicalPlan => plan - case _ => delegate.parsePlan(sqlText) - } - } - - override def parseExpression(sqlText: String): Expression = { - delegate.parseExpression(sqlText) - } - - override def parseTableIdentifier(sqlText: String): TableIdentifier = { - delegate.parseTableIdentifier(sqlText) - } - - override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = { - delegate.parseFunctionIdentifier(sqlText) - } - - override def parseMultipartIdentifier(sqlText: String): Seq[String] = { - delegate.parseMultipartIdentifier(sqlText) - } - - override def parseTableSchema(sqlText: String): StructType = { - delegate.parseTableSchema(sqlText) - } - - override def parseDataType(sqlText: String): DataType = { - delegate.parseDataType(sqlText) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala deleted file mode 100644 index cc4f0bd9ffc..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/parser/GlutenClickhouseSqlParser.scala +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.parser - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.catalyst.parser.ParserInterface -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.types.{DataType, StructType} - -class GlutenClickhouseSqlParser(spark: SparkSession, delegate: ParserInterface) - extends GlutenClickhouseSqlParserBase { - - override def parsePlan(sqlText: String): LogicalPlan = - parse(sqlText) { - parser => - astBuilder.visit(parser.singleStatement()) match { - case plan: LogicalPlan => plan - case _ => delegate.parsePlan(sqlText) - } - } - - override def parseExpression(sqlText: String): Expression = { - delegate.parseExpression(sqlText) - } - - override def parseTableIdentifier(sqlText: String): TableIdentifier = { - delegate.parseTableIdentifier(sqlText) - } - - override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = { - delegate.parseFunctionIdentifier(sqlText) - } - - override def parseMultipartIdentifier(sqlText: String): Seq[String] = { - delegate.parseMultipartIdentifier(sqlText) - } - - override def parseTableSchema(sqlText: String): StructType = { - delegate.parseTableSchema(sqlText) - } - - override def parseDataType(sqlText: String): DataType = { - delegate.parseDataType(sqlText) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/sql/shims/delta20/Delta20ShimProvider.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/sql/shims/delta20/Delta20ShimProvider.scala deleted file mode 100644 index 65efe745fb1..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/sql/shims/delta20/Delta20ShimProvider.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.sql.shims.delta20 - -import org.apache.gluten.sql.shims.{DeltaShimProvider, DeltaShims} - -class Delta20ShimProvider extends DeltaShimProvider { - - override def createShim: DeltaShims = { - new Delta20Shims() - } - -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/sql/shims/delta20/Delta20Shims.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/sql/shims/delta20/Delta20Shims.scala deleted file mode 100644 index 2348586e406..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/gluten/sql/shims/delta20/Delta20Shims.scala +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.sql.shims.delta20 - -import org.apache.gluten.sql.shims.DeltaShims - -class Delta20Shims extends DeltaShims {} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala deleted file mode 100644 index 588d73fe812..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/ClickhouseOptimisticTransaction.scala +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta - -import org.apache.gluten.backendsapi.clickhouse.CHConfig - -import org.apache.spark.SparkException -import org.apache.spark.sql.Dataset -import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.delta.constraints.{Constraint, Constraints} -import org.apache.spark.sql.delta.files.MergeTreeDelayedCommitProtocol -import org.apache.spark.sql.delta.schema.InvariantViolationException -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.execution.SQLExecution -import org.apache.spark.sql.execution.datasources.{BasicWriteJobStatsTracker, FileFormatWriter, GlutenWriterColumnarRules, WriteJobStatsTracker} -import org.apache.spark.sql.execution.datasources.v1.MergeTreeWriterInjects -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.util.{Clock, SerializableConfiguration} - -import org.apache.commons.lang3.exception.ExceptionUtils - -import scala.collection.mutable.ListBuffer - -class ClickhouseOptimisticTransaction( - override val deltaLog: DeltaLog, - override val snapshot: Snapshot)(implicit override val clock: Clock) - extends OptimisticTransaction(deltaLog, snapshot) { - - def this(deltaLog: DeltaLog, snapshotOpt: Option[Snapshot] = None)(implicit clock: Clock) { - this( - deltaLog, - snapshotOpt.getOrElse(deltaLog.update()) - ) - } - - override def writeFiles( - inputData: Dataset[_], - writeOptions: Option[DeltaOptions], - additionalConstraints: Seq[Constraint]): Seq[FileAction] = { - if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { - hasWritten = true - - val spark = inputData.sparkSession - val (data, partitionSchema) = performCDCPartition(inputData) - val outputPath = deltaLog.dataPath - - val (queryExecution, output, generatedColumnConstraints, _) = - normalizeData(deltaLog, data) - - val tableV2 = ClickHouseTableV2.getTable(deltaLog) - val committer = - new MergeTreeDelayedCommitProtocol( - outputPath.toString, - None, - tableV2.dataBaseName, - tableV2.tableName) - - // val (optionalStatsTracker, _) = - // getOptionalStatsTrackerAndStatsCollection(output, outputPath, partitionSchema, data) - val (optionalStatsTracker, _) = (None, None) - - val constraints = - Constraints.getAll(metadata, spark) ++ generatedColumnConstraints ++ additionalConstraints - - SQLExecution.withNewExecutionId(queryExecution, Option("deltaTransactionalWrite")) { - val queryPlan = queryExecution.executedPlan - val (newQueryPlan, newOutput) = - MergeTreeWriterInjects.insertFakeRowAdaptor(queryPlan, output) - val outputSpec = FileFormatWriter.OutputSpec(outputPath.toString, Map.empty, newOutput) - val partitioningColumns = getPartitioningColumns(partitionSchema, newOutput) - val statsTrackers: ListBuffer[WriteJobStatsTracker] = ListBuffer() - - if (spark.conf.get(DeltaSQLConf.DELTA_HISTORY_METRICS_ENABLED)) { - val basicWriteJobStatsTracker = new BasicWriteJobStatsTracker( - new SerializableConfiguration(deltaLog.newDeltaHadoopConf()), - BasicWriteJobStatsTracker.metrics) - // registerSQLMetrics(spark, basicWriteJobStatsTracker.driverSideMetrics) - statsTrackers.append(basicWriteJobStatsTracker) - } - - // Retain only a minimal selection of Spark writer options to avoid any potential - // compatibility issues - var options = writeOptions match { - case None => Map.empty[String, String] - case Some(writeOptions) => - writeOptions.options - .filterKeys { - key => - key.equalsIgnoreCase(DeltaOptions.MAX_RECORDS_PER_FILE) || - key.equalsIgnoreCase(DeltaOptions.COMPRESSION) - } - .map(identity) - } - - spark.conf.getAll.foreach( - entry => { - if ( - CHConfig.startWithSettingsPrefix(entry._1) - || entry._1.equalsIgnoreCase(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE.key) - ) { - options += (entry._1 -> entry._2) - } - }) - - try { - val format = tableV2.getFileFormat(metadata) - GlutenWriterColumnarRules.injectSparkLocalProperty(spark, Some(format.shortName()), None) - FileFormatWriter.write( - sparkSession = spark, - plan = newQueryPlan, - fileFormat = format, - // formats. - committer = committer, - outputSpec = outputSpec, - // scalastyle:off deltahadoopconfiguration - hadoopConf = spark.sessionState - .newHadoopConfWithOptions(metadata.configuration ++ deltaLog.options), - // scalastyle:on deltahadoopconfiguration - partitionColumns = partitioningColumns, - bucketSpec = - tableV2.normalizedBucketSpec(output.map(_.name), spark.sessionState.conf.resolver), - statsTrackers = optionalStatsTracker.toSeq ++ statsTrackers, - options = options - ) - } catch { - case s: SparkException => - // Pull an InvariantViolationException up to the top level if it was the root cause. - val violationException = ExceptionUtils.getRootCause(s) - if (violationException.isInstanceOf[InvariantViolationException]) { - throw violationException - } else { - throw s - } - } finally { - GlutenWriterColumnarRules.injectSparkLocalProperty(spark, None, None) - } - } - committer.addedStatuses.toSeq ++ committer.changeFiles - } else { - // TODO: support native delta parquet write - // 1. insert FakeRowAdaptor - // 2. DeltaInvariantCheckerExec transform - // 3. DeltaTaskStatisticsTracker collect null count / min values / max values - // 4. set the parameters 'staticPartitionWriteOnly', 'isNativeApplicable', - // 'nativeFormat' in the LocalProperty of the sparkcontext - super.writeFiles(inputData, writeOptions, additionalConstraints) - } - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala deleted file mode 100644 index 4ffa2e8415a..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/DeltaAdapter.scala +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import org.apache.spark.sql.delta.stats.DeltaScan - -object DeltaAdapter extends DeltaAdapterTrait { - override def snapshot(deltaLog: DeltaLog): Snapshot = deltaLog.snapshot - - override def snapshotFilesForScan( - snapshot: Snapshot, - projection: Seq[Attribute], - filters: Seq[Expression], - keepNumRecords: Boolean): DeltaScan = { - snapshot.filesForScan(projection, filters, keepNumRecords) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala deleted file mode 100644 index 57c6c8550fc..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/DeltaLog.scala +++ /dev/null @@ -1,794 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta - -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{Resolver, UnresolvedAttribute} -import org.apache.spark.sql.catalyst.catalog.{BucketSpec, CatalogTable} -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Cast, Expression, Literal} -import org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper -import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.commands.cdc.CDCReader -import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeLogFileIndex} -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.schema.{SchemaMergingUtils, SchemaUtils} -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.storage.LogStoreProvider -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.datasources._ -import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation} -import org.apache.spark.sql.types.{StructField, StructType} -import org.apache.spark.sql.util.CaseInsensitiveStringMap -import org.apache.spark.util.{Clock, SystemClock} - -import com.databricks.spark.util.TagDefinitions._ -import com.google.common.cache.{CacheBuilder, RemovalNotification} -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, Path} - -// scalastyle:off import.ordering.noEmptyLine -import java.io.File -import java.lang.ref.WeakReference -import java.util.concurrent.TimeUnit -import java.util.concurrent.locks.ReentrantLock - -import scala.collection.JavaConverters._ -import scala.collection.mutable -import scala.util.Try -import scala.util.control.NonFatal - -// This class is copied from Delta 2.0.1 because it has a private constructor, -// which makes it impossible to extend - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.0.1 It is modified to overcome the following issues: - * 1. return ClickhouseOptimisticTransaction 2. return DeltaMergeTreeFileFormat - */ - -/** - * Used to query the current state of the log as well as modify it by adding new atomic collections - * of actions. - * - * Internally, this class implements an optimistic concurrency control algorithm to handle multiple - * readers or writers. Any single read is guaranteed to see a consistent snapshot of the table. - */ -class DeltaLog private ( - val logPath: Path, - val dataPath: Path, - val options: Map[String, String], - val clock: Clock -) extends Checkpoints - with MetadataCleanup - with LogStoreProvider - with SnapshotManagement - with DeltaFileFormat - with ReadChecksum { - import org.apache.spark.sql.delta.util.FileNames._ - - import DeltaLog._ - - implicit private lazy val _clock = clock - - protected def spark = SparkSession.active - - /** - * Keep a reference to `SparkContext` used to create `DeltaLog`. `DeltaLog` cannot be used when - * `SparkContext` is stopped. We keep the reference so that we can check whether the cache is - * still valid and drop invalid `DeltaLog`` objects. - */ - private val sparkContext = new WeakReference(spark.sparkContext) - - /** - * Returns the Hadoop [[Configuration]] object which can be used to access the file system. All - * Delta code should use this method to create the Hadoop [[Configuration]] object, so that the - * hadoop file system configurations specified in DataFrame options will come into effect. - */ - // scalastyle:off deltahadoopconfiguration - final def newDeltaHadoopConf(): Configuration = - spark.sessionState.newHadoopConfWithOptions(options) - // scalastyle:on deltahadoopconfiguration - - /** Used to read and write physical log files and checkpoints. */ - lazy val store = createLogStore(spark) - - /** Use ReentrantLock to allow us to call `lockInterruptibly` */ - protected val deltaLogLock = new ReentrantLock() - - /** Delta History Manager containing version and commit history. */ - lazy val history = new DeltaHistoryManager( - this, - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_HISTORY_PAR_SEARCH_THRESHOLD)) - - /* --------------- * - | Configuration | - * --------------- */ - - /** - * The max lineage length of a Snapshot before Delta forces to build a Snapshot from scratch. - * Delta will build a Snapshot on top of the previous one if it doesn't see a checkpoint. However, - * there is a race condition that when two writers are writing at the same time, a writer may fail - * to pick up checkpoints written by another one, and the lineage will grow and finally cause - * StackOverflowError. Hence we have to force to build a Snapshot from scratch when the lineage - * length is too large to avoid hitting StackOverflowError. - */ - def maxSnapshotLineageLength: Int = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_MAX_SNAPSHOT_LINEAGE_LENGTH) - - /** How long to keep around logically deleted files before physically deleting them. */ - private[delta] def tombstoneRetentionMillis: Long = - DeltaConfigs.getMilliSeconds(DeltaConfigs.TOMBSTONE_RETENTION.fromMetaData(metadata)) - - // TODO: There is a race here where files could get dropped when increasing the - // retention interval... - protected def metadata = if (snapshot == null) Metadata() else snapshot.metadata - - /** - * Tombstones before this timestamp will be dropped from the state and the files can be garbage - * collected. - */ - def minFileRetentionTimestamp: Long = { - // TODO (Fred): Get rid of this FrameProfiler record once SC-94033 is addressed - recordFrameProfile("Delta", "DeltaLog.minFileRetentionTimestamp") { - clock.getTimeMillis() - tombstoneRetentionMillis - } - } - - /** - * [[SetTransaction]]s before this timestamp will be considered expired and dropped from the - * state, but no files will be deleted. - */ - def minSetTransactionRetentionTimestamp: Option[Long] = { - val intervalOpt = DeltaConfigs.TRANSACTION_ID_RETENTION_DURATION.fromMetaData(metadata) - - if (intervalOpt.isDefined) { - Some(clock.getTimeMillis() - DeltaConfigs.getMilliSeconds(intervalOpt.get)) - } else { - None - } - } - - /** - * Checks whether this table only accepts appends. If so it will throw an error in operations that - * can remove data such as DELETE/UPDATE/MERGE. - */ - def assertRemovable(): Unit = { - if (DeltaConfigs.IS_APPEND_ONLY.fromMetaData(metadata)) { - throw DeltaErrors.modifyAppendOnlyTableException(metadata.name) - } - } - - /** The unique identifier for this table. */ - def tableId: String = metadata.id - - /** - * Combines the tableId with the path of the table to ensure uniqueness. Normally `tableId` should - * be globally unique, but nothing stops users from copying a Delta table directly to a separate - * location, where the transaction log is copied directly, causing the tableIds to match. When - * users mutate the copied table, and then try to perform some checks joining the two tables, - * optimizations that depend on `tableId` alone may not be correct. Hence we use a composite id. - */ - private[delta] def compositeId: (String, Path) = tableId -> dataPath - - /** - * Run `body` inside `deltaLogLock` lock using `lockInterruptibly` so that the thread can be - * interrupted when waiting for the lock. - */ - def lockInterruptibly[T](body: => T): T = { - deltaLogLock.lockInterruptibly() - try { - body - } finally { - deltaLogLock.unlock() - } - } - - /* ------------------ * - | Delta Management | - * ------------------ */ - - /** - * Returns a new [[OptimisticTransaction]] that can be used to read the current state of the log - * and then commit updates. The reads and updates will be checked for logical conflicts with any - * concurrent writes to the log. - * - * Note that all reads in a transaction must go through the returned transaction object, and not - * directly to the [[DeltaLog]] otherwise they will not be checked for conflicts. - */ - def startTransaction(): OptimisticTransaction = { - update() - // --- modified start - new ClickhouseOptimisticTransaction(this, None) - // --- modified end - } - - /** - * Execute a piece of code within a new [[OptimisticTransaction]]. Reads/write sets will be - * recorded for this table, and all other tables will be read at a snapshot that is pinned on the - * first access. - * - * @note - * This uses thread-local variable to make the active transaction visible. So do not use - * multi-threaded code in the provided thunk. - */ - def withNewTransaction[T](thunk: OptimisticTransaction => T): T = { - try { - val txn = startTransaction() - OptimisticTransaction.setActive(txn) - thunk(txn) - } finally { - OptimisticTransaction.clearActive() - } - } - - /** - * Upgrade the table's protocol version, by default to the maximum recognized reader and writer - * versions in this DBR release. - */ - def upgradeProtocol(newVersion: Protocol = Protocol()): Unit = { - val currentVersion = snapshot.protocol - if ( - newVersion.minReaderVersion == currentVersion.minReaderVersion && - newVersion.minWriterVersion == currentVersion.minWriterVersion - ) { - logConsole(s"Table $dataPath is already at protocol version $newVersion.") - return - } - - val txn = startTransaction() - try { - SchemaMergingUtils.checkColumnNameDuplication(txn.metadata.schema, "in the table schema") - } catch { - case e: AnalysisException => - throw DeltaErrors.duplicateColumnsOnUpdateTable(e) - } - txn.commit(Seq(newVersion), DeltaOperations.UpgradeProtocol(newVersion)) - logConsole(s"Upgraded table at $dataPath to $newVersion.") - } - - /** - * Get all actions starting from "startVersion" (inclusive). If `startVersion` doesn't exist, - * return an empty Iterator. - */ - def getChanges( - startVersion: Long, - failOnDataLoss: Boolean = false): Iterator[(Long, Seq[Action])] = { - val hadoopConf = newDeltaHadoopConf() - val deltas = store - .listFrom(deltaFile(logPath, startVersion), hadoopConf) - .filter(f => isDeltaFile(f.getPath)) - // Subtract 1 to ensure that we have the same check for the inclusive startVersion - var lastSeenVersion = startVersion - 1 - deltas.map { - status => - val p = status.getPath - val version = deltaVersion(p) - if (failOnDataLoss && version > lastSeenVersion + 1) { - throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) - } - lastSeenVersion = version - (version, store.read(p, hadoopConf).map(Action.fromJson)) - } - } - - /** - * Get access to all actions starting from "startVersion" (inclusive) via [[FileStatus]]. If - * `startVersion` doesn't exist, return an empty Iterator. - */ - def getChangeLogFiles( - startVersion: Long, - failOnDataLoss: Boolean = false): Iterator[(Long, FileStatus)] = { - val deltas = store - .listFrom(deltaFile(logPath, startVersion), newDeltaHadoopConf()) - .filter(f => isDeltaFile(f.getPath)) - // Subtract 1 to ensure that we have the same check for the inclusive startVersion - var lastSeenVersion = startVersion - 1 - deltas.map { - status => - val version = deltaVersion(status.getPath) - if (failOnDataLoss && version > lastSeenVersion + 1) { - throw DeltaErrors.failOnDataLossException(lastSeenVersion + 1, version) - } - lastSeenVersion = version - (version, status) - } - } - - /* --------------------- * - | Protocol validation | - * --------------------- */ - - /** - * Asserts that the client is up to date with the protocol and allowed to read the table that is - * using the given `protocol`. - */ - def protocolRead(protocol: Protocol): Unit = { - val supportedReaderVersion = - Action.supportedProtocolVersion(Some(spark.sessionState.conf)).minReaderVersion - if (protocol != null && supportedReaderVersion < protocol.minReaderVersion) { - recordDeltaEvent( - this, - "delta.protocol.failure.read", - data = Map( - "clientVersion" -> supportedReaderVersion, - "minReaderVersion" -> protocol.minReaderVersion)) - throw new InvalidProtocolVersionException - } - } - - /** - * Asserts that the client is up to date with the protocol and allowed to write to the table that - * is using the given `protocol`. - */ - def protocolWrite(protocol: Protocol, logUpgradeMessage: Boolean = true): Unit = { - val supportedWriterVersion = - Action.supportedProtocolVersion(Some(spark.sessionState.conf)).minWriterVersion - if (protocol != null && supportedWriterVersion < protocol.minWriterVersion) { - recordDeltaEvent( - this, - "delta.protocol.failure.write", - data = Map( - "clientVersion" -> supportedWriterVersion, - "minWriterVersion" -> protocol.minWriterVersion)) - throw new InvalidProtocolVersionException - } - } - - /* ---------------------------------------- * - | Log Directory Management and Retention | - * ---------------------------------------- */ - - /** Whether a Delta table exists at this directory. */ - def tableExists: Boolean = snapshot.version >= 0 - - def isSameLogAs(otherLog: DeltaLog): Boolean = this.compositeId == otherLog.compositeId - - /** Creates the log directory if it does not exist. */ - def ensureLogDirectoryExist(): Unit = { - val fs = logPath.getFileSystem(newDeltaHadoopConf()) - if (!fs.exists(logPath)) { - if (!fs.mkdirs(logPath)) { - throw DeltaErrors.cannotCreateLogPathException(logPath.toString) - } - } - } - - /** - * Create the log directory. Unlike `ensureLogDirectoryExist`, this method doesn't check whether - * the log directory exists and it will ignore the return value of `mkdirs`. - */ - def createLogDirectory(): Unit = { - logPath.getFileSystem(newDeltaHadoopConf()).mkdirs(logPath) - } - - /* ------------ * - | Integration | - * ------------ */ - - /** - * Returns a [[org.apache.spark.sql.DataFrame]] containing the new files within the specified - * version range. - */ - def createDataFrame( - snapshot: Snapshot, - addFiles: Seq[AddFile], - isStreaming: Boolean = false, - actionTypeOpt: Option[String] = None): DataFrame = { - val actionType = actionTypeOpt.getOrElse(if (isStreaming) "streaming" else "batch") - val fileIndex = new TahoeBatchFileIndex(spark, actionType, addFiles, this, dataPath, snapshot) - - val relation = HadoopFsRelation( - fileIndex, - partitionSchema = - DeltaColumnMapping.dropColumnMappingMetadata(snapshot.metadata.partitionSchema), - // We pass all table columns as `dataSchema` so that Spark will preserve the partition column - // locations. Otherwise, for any partition columns not in `dataSchema`, Spark would just - // append them to the end of `dataSchema`. - dataSchema = DeltaColumnMapping.dropColumnMappingMetadata( - ColumnWithDefaultExprUtils.removeDefaultExpressions(snapshot.metadata.schema)), - bucketSpec = None, - snapshot.deltaLog.fileFormat(snapshot.metadata), - snapshot.metadata.format.options - )(spark) - - Dataset.ofRows(spark, LogicalRelation(relation, isStreaming = isStreaming)) - } - - /** - * Returns a [[BaseRelation]] that contains all of the data present in the table. This relation - * will be continually updated as files are added or removed from the table. However, new - * [[BaseRelation]] must be requested in order to see changes to the schema. - */ - def createRelation( - partitionFilters: Seq[Expression] = Nil, - snapshotToUseOpt: Option[Snapshot] = None, - isTimeTravelQuery: Boolean = false, - cdcOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty): BaseRelation = { - - /** Used to link the files present in the table into the query planner. */ - val snapshotToUse = snapshotToUseOpt.getOrElse(snapshot) - if (snapshotToUse.version < 0) { - // A negative version here means the dataPath is an empty directory. Read query should error - // out in this case. - throw DeltaErrors.pathNotExistsException(dataPath.toString) - } - - // For CDC we have to return the relation that represents the change data instead of actual - // data. - if (!cdcOptions.isEmpty) { - recordDeltaEvent(this, "delta.cdf.read", data = cdcOptions.asCaseSensitiveMap()) - return CDCReader.getCDCRelation( - spark, - this, - snapshotToUse, - partitionFilters, - spark.sessionState.conf, - cdcOptions) - } - - val fileIndex = - TahoeLogFileIndex(spark, this, dataPath, snapshotToUse, partitionFilters, isTimeTravelQuery) - // --- modified start - val bucketSpec: Option[BucketSpec] = - if (ClickHouseConfig.isMergeTreeFormatEngine(snapshotToUse.metadata.configuration)) { - ClickHouseTableV2.getTable(this).bucketOption - } else { - None - } - new DeltaHadoopFsRelation( - fileIndex, - partitionSchema = - DeltaColumnMapping.dropColumnMappingMetadata(snapshotToUse.metadata.partitionSchema), - // We pass all table columns as `dataSchema` so that Spark will preserve the partition column - // locations. Otherwise, for any partition columns not in `dataSchema`, Spark would just - // append them to the end of `dataSchema` - dataSchema = DeltaColumnMapping.dropColumnMappingMetadata( - ColumnWithDefaultExprUtils.removeDefaultExpressions( - SchemaUtils.dropNullTypeColumns(snapshotToUse.metadata.schema))), - bucketSpec = bucketSpec, - fileFormat(snapshotToUse.metadata), - // `metadata.format.options` is not set today. Even if we support it in future, we shouldn't - // store any file system options since they may contain credentials. Hence, it will never - // conflict with `DeltaLog.options`. - snapshotToUse.metadata.format.options ++ options - )( - spark, - this - ) - // --- modified end - } - - override def fileFormat(metadata: Metadata = metadata): FileFormat = { - // --- modified start - if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { - ClickHouseTableV2.getTable(this).getFileFormat(metadata) - } else { - super.fileFormat(metadata) - } - // --- modified end - } -} - -object DeltaLog extends DeltaLogging { - // --- modified start - @SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) - private class DeltaHadoopFsRelation( - location: FileIndex, - partitionSchema: StructType, - // The top-level columns in `dataSchema` should match the actual physical file schema, - // otherwise the ORC data source may not work with the by-ordinal mode. - dataSchema: StructType, - bucketSpec: Option[BucketSpec], - fileFormat: FileFormat, - options: Map[String, String])(sparkSession: SparkSession, deltaLog: DeltaLog) - extends HadoopFsRelation( - location, - partitionSchema, - dataSchema, - bucketSpec, - fileFormat, - options)(sparkSession) - with InsertableRelation { - def insert(data: DataFrame, overwrite: Boolean): Unit = { - val mode = if (overwrite) SaveMode.Overwrite else SaveMode.Append - WriteIntoDelta( - deltaLog = deltaLog, - mode = mode, - new DeltaOptions(Map.empty[String, String], sparkSession.sessionState.conf), - partitionColumns = Seq.empty, - configuration = Map.empty, - data = data - ).run(sparkSession) - } - } - // --- modified end - - /** - * The key type of `DeltaLog` cache. It's a pair of the canonicalized table path and the file - * system options (options starting with "fs." prefix) passed into `DataFrameReader/Writer` - */ - private type DeltaLogCacheKey = (Path, Map[String, String]) - - /** - * We create only a single [[DeltaLog]] for any given `DeltaLogCacheKey` to avoid wasted work in - * reconstructing the log. - */ - private val deltaLogCache = { - val builder = CacheBuilder - .newBuilder() - .expireAfterAccess(60, TimeUnit.MINUTES) - .removalListener( - (removalNotification: RemovalNotification[DeltaLogCacheKey, DeltaLog]) => { - val log = removalNotification.getValue - try log.snapshot.uncache() - catch { - case _: java.lang.NullPointerException => - // Various layers will throw null pointer if the RDD is already gone. - } - }) - sys.props - .get("delta.log.cacheSize") - .flatMap(v => Try(v.toLong).toOption) - .foreach(builder.maximumSize) - builder.build[DeltaLogCacheKey, DeltaLog]() - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: String): DeltaLog = { - apply(spark, new Path(dataPath, "_delta_log"), Map.empty, new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: String, options: Map[String, String]): DeltaLog = { - apply(spark, new Path(dataPath, "_delta_log"), options, new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: File): DeltaLog = { - apply(spark, new Path(dataPath.getAbsolutePath, "_delta_log"), new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: Path): DeltaLog = { - apply(spark, new Path(dataPath, "_delta_log"), new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: Path, options: Map[String, String]): DeltaLog = { - apply(spark, new Path(dataPath, "_delta_log"), options, new SystemClock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: String, clock: Clock): DeltaLog = { - apply(spark, new Path(dataPath, "_delta_log"), clock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: File, clock: Clock): DeltaLog = { - apply(spark, new Path(dataPath.getAbsolutePath, "_delta_log"), clock) - } - - /** Helper for creating a log when it stored at the root of the data. */ - def forTable(spark: SparkSession, dataPath: Path, clock: Clock): DeltaLog = { - apply(spark, new Path(dataPath, "_delta_log"), clock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, tableName: TableIdentifier): DeltaLog = { - forTable(spark, tableName, new SystemClock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, table: CatalogTable): DeltaLog = { - forTable(spark, table, new SystemClock) - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, tableName: TableIdentifier, clock: Clock): DeltaLog = { - if (DeltaTableIdentifier.isDeltaPath(spark, tableName)) { - forTable(spark, new Path(tableName.table)) - } else { - forTable(spark, spark.sessionState.catalog.getTableMetadata(tableName), clock) - } - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, table: CatalogTable, clock: Clock): DeltaLog = { - val log = apply(spark, new Path(new Path(table.location), "_delta_log"), clock) - log - } - - /** Helper for creating a log for the table. */ - def forTable(spark: SparkSession, deltaTable: DeltaTableIdentifier): DeltaLog = { - if (deltaTable.path.isDefined) { - forTable(spark, deltaTable.path.get) - } else { - forTable(spark, deltaTable.table.get) - } - } - - private def apply(spark: SparkSession, rawPath: Path, clock: Clock = new SystemClock): DeltaLog = - apply(spark, rawPath, Map.empty, clock) - - private def apply( - spark: SparkSession, - rawPath: Path, - options: Map[String, String], - clock: Clock): DeltaLog = { - val fileSystemOptions: Map[String, String] = - if ( - spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) - ) { - // We pick up only file system options so that we don't pass any parquet or json options to - // the code that reads Delta transaction logs. - options.filterKeys(_.startsWith("fs.")).toMap - } else { - Map.empty - } - // scalastyle:off deltahadoopconfiguration - val hadoopConf = spark.sessionState.newHadoopConfWithOptions(fileSystemOptions) - // scalastyle:on deltahadoopconfiguration - var path = rawPath - val fs = path.getFileSystem(hadoopConf) - path = fs.makeQualified(path) - def createDeltaLog(): DeltaLog = recordDeltaOperation( - null, - "delta.log.create", - Map(TAG_TAHOE_PATH -> path.getParent.toString)) { - AnalysisHelper.allowInvokingTransformsInAnalyzer { - new DeltaLog( - logPath = path, - dataPath = path.getParent, - options = fileSystemOptions, - clock = clock - ) - } - } - def getDeltaLogFromCache(): DeltaLog = { - // The following cases will still create a new ActionLog even if there is a cached - // ActionLog using a different format path: - // - Different `scheme` - // - Different `authority` (e.g., different user tokens in the path) - // - Different mount point. - try { - deltaLogCache.get(path -> fileSystemOptions, () => createDeltaLog()) - } catch { - case e: com.google.common.util.concurrent.UncheckedExecutionException => - throw e.getCause - } - } - - val deltaLog = getDeltaLogFromCache() - if (Option(deltaLog.sparkContext.get).map(_.isStopped).getOrElse(true)) { - // Invalid the cached `DeltaLog` and create a new one because the `SparkContext` of the cached - // `DeltaLog` has been stopped. - deltaLogCache.invalidate(path -> fileSystemOptions) - getDeltaLogFromCache() - } else { - deltaLog - } - } - - /** Invalidate the cached DeltaLog object for the given `dataPath`. */ - def invalidateCache(spark: SparkSession, dataPath: Path): Unit = { - try { - val rawPath = new Path(dataPath, "_delta_log") - // scalastyle:off deltahadoopconfiguration - // This method cannot be called from DataFrameReader/Writer so it's safe to assume the user - // has set the correct file system configurations in the session configs. - val fs = rawPath.getFileSystem(spark.sessionState.newHadoopConf()) - // scalastyle:on deltahadoopconfiguration - val path = fs.makeQualified(rawPath) - - if ( - spark.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) - ) { - // We rely on the fact that accessing the key set doesn't modify the entry access time. See - // `CacheBuilder.expireAfterAccess`. - val keysToBeRemoved = mutable.ArrayBuffer[DeltaLogCacheKey]() - val iter = deltaLogCache.asMap().keySet().iterator() - while (iter.hasNext) { - val key = iter.next() - if (key._1 == path) { - keysToBeRemoved += key - } - } - deltaLogCache.invalidateAll(keysToBeRemoved.asJava) - } else { - deltaLogCache.invalidate(path -> Map.empty) - } - } catch { - case NonFatal(e) => logWarning(e.getMessage, e) - } - } - - def clearCache(): Unit = { - deltaLogCache.invalidateAll() - } - - /** Return the number of cached `DeltaLog`s. Exposing for testing */ - private[delta] def cacheSize: Long = { - deltaLogCache.size() - } - - /** - * Filters the given [[Dataset]] by the given `partitionFilters`, returning those that match. - * @param files - * The active files in the DeltaLog state, which contains the partition value information - * @param partitionFilters - * Filters on the partition columns - * @param partitionColumnPrefixes - * The path to the `partitionValues` column, if it's nested - */ - def filterFileList( - partitionSchema: StructType, - files: DataFrame, - partitionFilters: Seq[Expression], - partitionColumnPrefixes: Seq[String] = Nil): DataFrame = { - val rewrittenFilters = rewritePartitionFilters( - partitionSchema, - files.sparkSession.sessionState.conf.resolver, - partitionFilters, - partitionColumnPrefixes) - val expr = rewrittenFilters.reduceLeftOption(And).getOrElse(Literal.TrueLiteral) - val columnFilter = new Column(expr) - files.filter(columnFilter) - } - - /** - * Rewrite the given `partitionFilters` to be used for filtering partition values. We need to - * explicitly resolve the partitioning columns here because the partition columns are stored as - * keys of a Map type instead of attributes in the AddFile schema (below) and thus cannot be - * resolved automatically. - * - * @param partitionFilters - * Filters on the partition columns - * @param partitionColumnPrefixes - * The path to the `partitionValues` column, if it's nested - */ - def rewritePartitionFilters( - partitionSchema: StructType, - resolver: Resolver, - partitionFilters: Seq[Expression], - partitionColumnPrefixes: Seq[String] = Nil): Seq[Expression] = { - partitionFilters.map(_.transformUp { - case a: Attribute => - // If we have a special column name, e.g. `a.a`, then an UnresolvedAttribute returns - // the column name as '`a.a`' instead of 'a.a', therefore we need to strip the backticks. - val unquoted = a.name.stripPrefix("`").stripSuffix("`") - val partitionCol = partitionSchema.find(field => resolver(field.name, unquoted)) - partitionCol match { - case Some(f: StructField) => - val name = DeltaColumnMapping.getPhysicalName(f) - Cast( - UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", name)), - f.dataType) - case None => - // This should not be able to happen, but the case was present in the original code so - // we kept it to be safe. - log.error(s"Partition filter referenced column ${a.name} not in the partition schema") - UnresolvedAttribute(partitionColumnPrefixes ++ Seq("partitionValues", a.name)) - } - }) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/Snapshot.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/Snapshot.scala deleted file mode 100644 index 2e4d6bb2207..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/Snapshot.scala +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta - -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import org.apache.spark.sql.delta.actions._ -import org.apache.spark.sql.delta.actions.Action.logSchema -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.schema.SchemaUtils -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.stats.{DataSkippingReader, DeltaScan, FileSizeHistogram, StatisticsCollection} -import org.apache.spark.sql.delta.util.StateCache -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, LogicalRelation} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.{SerializableConfiguration, Utils} - -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} - -// scalastyle:off import.ordering.noEmptyLine -import java.net.URI - -import scala.collection.mutable - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.0.1. It is modified to overcome the following issues: - * 1. filesForScan() should return DeltaScan of AddMergeTreeParts instead of AddFile - */ - -/** - * An immutable snapshot of the state of the log at some delta version. Internally this class - * manages the replay of actions stored in checkpoint or delta files. - * - * After resolving any new actions, it caches the result and collects the following basic - * information to the driver: - * - Protocol Version - * - Metadata - * - Transaction state - * - * @param timestamp - * The timestamp of the latest commit in milliseconds. Can also be set to -1 if the timestamp of - * the commit is unknown or the table has not been initialized, i.e. `version = -1`. - */ -class Snapshot( - val path: Path, - val version: Long, - val logSegment: LogSegment, - val minFileRetentionTimestamp: Long, - val deltaLog: DeltaLog, - val timestamp: Long, - val checksumOpt: Option[VersionChecksum], - val minSetTransactionRetentionTimestamp: Option[Long] = None, - checkpointMetadataOpt: Option[CheckpointMetaData] = None) - extends StateCache - with StatisticsCollection - with DataSkippingReader - with DeltaLogging { - - // For implicits which re-use Encoder: - import SingleAction._ - import Snapshot._ - - protected def spark = SparkSession.active - - /** Snapshot to scan by the DeltaScanGenerator for metadata query optimizations */ - override val snapshotToScan: Snapshot = this - - protected def getNumPartitions: Int = { - spark.sessionState.conf - .getConf(DeltaSQLConf.DELTA_SNAPSHOT_PARTITIONS) - .getOrElse(Snapshot.defaultNumSnapshotPartitions) - } - - /** Performs validations during initialization */ - protected def init(): Unit = { - deltaLog.protocolRead(protocol) - SchemaUtils.recordUndefinedTypes(deltaLog, metadata.schema) - } - - // Reconstruct the state by applying deltas in order to the checkpoint. - // We partition by path as it is likely the bulk of the data is add/remove. - // Non-path based actions will be collocated to a single partition. - private def stateReconstruction: Dataset[SingleAction] = { - withDmqTag { - recordFrameProfile("Delta", "snapshot.stateReconstruction") { - val implicits = spark.implicits - - // for serializability - val localMinFileRetentionTimestamp = minFileRetentionTimestamp - val localMinSetTransactionRetentionTimestamp = minSetTransactionRetentionTimestamp - val localLogPath = path.toUri - - val hadoopConf = - spark.sparkContext.broadcast(new SerializableConfiguration(deltaLog.newDeltaHadoopConf())) - var wrapPath = false - - val canonicalizePath = DeltaUDF.stringStringUdf( - (filePath: String) => Snapshot.canonicalizePath(filePath, hadoopConf.value.value)) - - // Canonicalize the paths so we can repartition the actions correctly, but only rewrite the - // add/remove actions themselves after partitioning and sorting are complete. Otherwise, the - // optimizer can generate a really bad plan that re-evaluates _EVERY_ field of the rewritten - // struct(...) projection every time we touch _ANY_ field of the rewritten struct. - // - // NOTE: We sort by [[ACTION_SORT_COL_NAME]] (provided by [[loadActions]]), to ensure that - // actions are presented to InMemoryLogReplay in the ascending version order it expects. - val ADD_PATH_CANONICAL_COL_NAME = "add_path_canonical" - val REMOVE_PATH_CANONICAL_COL_NAME = "remove_path_canonical" - loadActions - .withColumn( - ADD_PATH_CANONICAL_COL_NAME, - when(col("add.path").isNotNull, canonicalizePath(col("add.path")))) - .withColumn( - REMOVE_PATH_CANONICAL_COL_NAME, - when(col("remove.path").isNotNull, canonicalizePath(col("remove.path")))) - .repartition( - getNumPartitions, - coalesce(col(ADD_PATH_CANONICAL_COL_NAME), col(REMOVE_PATH_CANONICAL_COL_NAME))) - .sortWithinPartitions(ACTION_SORT_COL_NAME) - .withColumn( - "add", - when( - col("add.path").isNotNull, - struct( - col(ADD_PATH_CANONICAL_COL_NAME).as("path"), - col("add.partitionValues"), - col("add.size"), - col("add.modificationTime"), - col("add.dataChange"), - col(ADD_STATS_TO_USE_COL_NAME).as("stats"), - col("add.tags") - ) - ) - ) - .withColumn( - "remove", - when( - col("remove.path").isNotNull, - col("remove").withField("path", col(REMOVE_PATH_CANONICAL_COL_NAME)))) - .as[SingleAction] - .mapPartitions { - iter => - val state: LogReplay = - new InMemoryLogReplay( - localMinFileRetentionTimestamp, - localMinSetTransactionRetentionTimestamp) - state.append(0, iter.map(_.unwrap)) - state.checkpoint.map(_.wrap) - } - } - } - } - - def redactedPath: String = - Utils.redact(spark.sessionState.conf.stringRedactionPattern, path.toUri.toString) - - private lazy val cachedState = withDmqTag { - cacheDS(stateReconstruction, s"Delta Table State #$version - $redactedPath") - } - - /** The current set of actions in this [[Snapshot]] as a typed Dataset. */ - def stateDS: Dataset[SingleAction] = withDmqTag { - cachedState.getDS - } - - /** The current set of actions in this [[Snapshot]] as plain Rows */ - def stateDF: DataFrame = withDmqTag { - cachedState.getDF - } - - /** Helper method to log missing actions when state reconstruction checks are not enabled */ - protected def logMissingActionWarning(action: String): Unit = { - logWarning(s""" - |Found no $action in computed state, setting it to defaults. State reconstruction - |validation was turned off. To turn it back on set - |${DeltaSQLConf.DELTA_STATE_RECONSTRUCTION_VALIDATION_ENABLED.key} to "true" - """.stripMargin) - } - - /** A Map of alias to aggregations which needs to be done to calculate the `computedState` */ - protected def aggregationsToComputeState: Map[String, Column] = { - val implicits = spark.implicits - import implicits._ - Map( - "protocol" -> last($"protocol", ignoreNulls = true), - "metadata" -> last($"metaData", ignoreNulls = true), - "setTransactions" -> collect_set($"txn"), - // sum may return null for empty data set. - "sizeInBytes" -> coalesce(sum($"add.size"), lit(0L)), - "numOfFiles" -> count($"add"), - "numOfMetadata" -> count($"metaData"), - "numOfProtocol" -> count($"protocol"), - "numOfRemoves" -> count($"remove"), - "numOfSetTransactions" -> count($"txn"), - "fileSizeHistogram" -> lit(null).cast(FileSizeHistogram.schema) - ) - } - - /** - * Computes some statistics around the transaction log, therefore on the actions made on this - * Delta table. - */ - protected lazy val computedState: State = { - withStatusCode("DELTA", s"Compute snapshot for version: $version") { - withDmqTag { - recordFrameProfile("Delta", "snapshot.computedState") { - val startTime = System.nanoTime() - val aggregations = - aggregationsToComputeState.map { case (alias, agg) => agg.as(alias) }.toSeq - val _computedState = stateDF.select(aggregations: _*).as[State](stateEncoder).first() - val stateReconstructionCheck = spark.sessionState.conf.getConf( - DeltaSQLConf.DELTA_STATE_RECONSTRUCTION_VALIDATION_ENABLED) - if (_computedState.protocol == null) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.missingAction", - data = - Map("version" -> version.toString, "action" -> "Protocol", "source" -> "Snapshot")) - if (stateReconstructionCheck) { - throw DeltaErrors.actionNotFoundException("protocol", version) - } - } - if (_computedState.metadata == null) { - recordDeltaEvent( - deltaLog, - opType = "delta.assertions.missingAction", - data = - Map("version" -> version.toString, "action" -> "Metadata", "source" -> "Metadata")) - if (stateReconstructionCheck) { - throw DeltaErrors.actionNotFoundException("metadata", version) - } - logMissingActionWarning("metadata") - _computedState.copy(metadata = Metadata()) - } else { - _computedState - } - } - } - } - } - - def protocol: Protocol = computedState.protocol - def metadata: Metadata = computedState.metadata - def setTransactions: Seq[SetTransaction] = computedState.setTransactions - def sizeInBytes: Long = computedState.sizeInBytes - def numOfFiles: Long = computedState.numOfFiles - def fileSizeHistogram: Option[FileSizeHistogram] = computedState.fileSizeHistogram - def numOfMetadata: Long = computedState.numOfMetadata - def numOfProtocol: Long = computedState.numOfProtocol - def numOfRemoves: Long = computedState.numOfRemoves - def numOfSetTransactions: Long = computedState.numOfSetTransactions - - /** - * Computes all the information that is needed by the checksum for the current snapshot. May kick - * off state reconstruction if needed by any of the underlying fields. Note that it's safe to set - * txnId to none, since the snapshot doesn't always have a txn attached. E.g. if a snapshot is - * created by reading a checkpoint, then no txnId is present. - */ - def computeChecksum: VersionChecksum = VersionChecksum( - tableSizeBytes = sizeInBytes, - numFiles = numOfFiles, - numMetadata = numOfMetadata, - numProtocol = numOfProtocol, - protocol = protocol, - metadata = metadata, - histogramOpt = fileSizeHistogram, - txnId = None - ) - - /** A map to look up transaction version by appId. */ - lazy val transactions: Map[String, Long] = setTransactions.map(t => t.appId -> t.version).toMap - - // Here we need to bypass the ACL checks for SELECT anonymous function permissions. - /** All of the files present in this [[Snapshot]]. */ - def allFiles: Dataset[AddFile] = { - val implicits = spark.implicits - import implicits._ - stateDS.where("add IS NOT NULL").select($"add".as[AddFile]) - } - - /** All unexpired tombstones. */ - def tombstones: Dataset[RemoveFile] = { - val implicits = spark.implicits - import implicits._ - stateDS.where("remove IS NOT NULL").select($"remove".as[RemoveFile]) - } - - /** Returns the schema of the table. */ - def schema: StructType = metadata.schema - - /** Returns the data schema of the table, the schema of the columns written out to file. */ - def dataSchema: StructType = metadata.dataSchema - - /** Number of columns to collect stats on for data skipping */ - lazy val numIndexedCols: Int = DeltaConfigs.DATA_SKIPPING_NUM_INDEXED_COLS.fromMetaData(metadata) - - /** Return the set of properties of the table. */ - def getProperties: mutable.HashMap[String, String] = { - val base = new mutable.HashMap[String, String]() - metadata.configuration.foreach { - case (k, v) => - if (k != "path") { - base.put(k, v) - } - } - base.put(Protocol.MIN_READER_VERSION_PROP, protocol.minReaderVersion.toString) - base.put(Protocol.MIN_WRITER_VERSION_PROP, protocol.minWriterVersion.toString) - base - } - - // Given the list of files from `LogSegment`, create respective file indices to help create - // a DataFrame and short-circuit the many file existence and partition schema inference checks - // that exist in DataSource.resolveRelation(). - protected lazy val deltaFileIndexOpt: Option[DeltaLogFileIndex] = { - assertLogFilesBelongToTable(path, logSegment.deltas) - DeltaLogFileIndex(DeltaLogFileIndex.COMMIT_FILE_FORMAT, logSegment.deltas) - } - - protected lazy val checkpointFileIndexOpt: Option[DeltaLogFileIndex] = { - assertLogFilesBelongToTable(path, logSegment.checkpoint) - DeltaLogFileIndex(DeltaLogFileIndex.CHECKPOINT_FILE_FORMAT, logSegment.checkpoint) - } - - def getCheckpointMetadataOpt: Option[CheckpointMetaData] = checkpointMetadataOpt - - def deltaFileSizeInBytes(): Long = deltaFileIndexOpt.map(_.sizeInBytes).getOrElse(0L) - def checkpointSizeInBytes(): Long = checkpointFileIndexOpt.map(_.sizeInBytes).getOrElse(0L) - - protected lazy val fileIndices: Seq[DeltaLogFileIndex] = { - checkpointFileIndexOpt.toSeq ++ deltaFileIndexOpt.toSeq - } - - /** Creates a LogicalRelation with the given schema from a DeltaLogFileIndex. */ - protected def indexToRelation( - index: DeltaLogFileIndex, - schema: StructType = logSchema): LogicalRelation = { - val fsRelation = - HadoopFsRelation(index, index.partitionSchema, schema, None, index.format, deltaLog.options)( - spark) - LogicalRelation(fsRelation) - } - - /** - * Loads the file indices into a DataFrame that can be used for LogReplay. - * - * In addition to the usual nested columns provided by the SingleAction schema, it should provide - * two additional columns to simplify the log replay process: [[ACTION_SORT_COL_NAME]] (which, - * when sorted in ascending order, will order older actions before newer ones, as required by - * [[InMemoryLogReplay]]); and [[ADD_STATS_TO_USE_COL_NAME]] (to handle certain combinations of - * config settings for delta.checkpoint.writeStatsAsJson and delta.checkpoint.writeStatsAsStruct). - */ - protected def loadActions: DataFrame = { - val dfs = fileIndices.map(index => Dataset.ofRows(spark, indexToRelation(index))) - dfs - .reduceOption(_.union(_)) - .getOrElse(emptyDF) - .withColumn(ACTION_SORT_COL_NAME, input_file_name()) - .withColumn(ADD_STATS_TO_USE_COL_NAME, col("add.stats")) - } - - protected def emptyDF: DataFrame = - spark.createDataFrame(spark.sparkContext.emptyRDD[Row], logSchema) - - override def logInfo(msg: => String): Unit = { - super.logInfo(s"[tableId=${deltaLog.tableId}] " + msg) - } - - override def logWarning(msg: => String): Unit = { - super.logWarning(s"[tableId=${deltaLog.tableId}] " + msg) - } - - override def logWarning(msg: => String, throwable: Throwable): Unit = { - super.logWarning(s"[tableId=${deltaLog.tableId}] " + msg, throwable) - } - - override def logError(msg: => String): Unit = { - super.logError(s"[tableId=${deltaLog.tableId}] " + msg) - } - - override def logError(msg: => String, throwable: Throwable): Unit = { - super.logError(s"[tableId=${deltaLog.tableId}] " + msg, throwable) - } - - override def toString: String = - s"${getClass.getSimpleName}(path=$path, version=$version, metadata=$metadata, " + - s"logSegment=$logSegment, checksumOpt=$checksumOpt)" - - // --- modified start - override def filesForScan( - projection: Seq[Attribute], - filters: Seq[Expression], - keepNumRecords: Boolean): DeltaScan = { - val deltaScan = ClickhouseSnapshot.deltaScanCache.get( - FilterExprsAsKey(path, ClickhouseSnapshot.genSnapshotId(this), filters, None), - () => { - super.filesForScan(projection, filters, keepNumRecords) - }) - - replaceWithAddMergeTreeParts(deltaScan) - } - - private def replaceWithAddMergeTreeParts(deltaScan: DeltaScan) = { - if (ClickHouseConfig.isMergeTreeFormatEngine(metadata.configuration)) { - DeltaScan.apply( - deltaScan.version, - deltaScan.files - .map( - addFile => { - val addFileAsKey = AddFileAsKey(addFile) - - val ret = ClickhouseSnapshot.addFileToAddMTPCache.get(addFileAsKey) - // this is for later use - ClickhouseSnapshot.pathToAddMTPCache.put(ret.fullPartPath(), ret) - ret - }), - deltaScan.total, - deltaScan.partition, - deltaScan.scanned - )( - deltaScan.scannedSnapshot, - deltaScan.partitionFilters, - deltaScan.dataFilters, - deltaScan.unusedFilters, - deltaScan.projection, - deltaScan.scanDurationMs, - deltaScan.dataSkippingType - ) - } else { - deltaScan - } - } - // --- modified end - - logInfo(s"Created snapshot $this") - init() -} -object Snapshot extends DeltaLogging { - - // Used by [[loadActions]] and [[stateReconstruction]] - val ACTION_SORT_COL_NAME = "action_sort_column" - val ADD_STATS_TO_USE_COL_NAME = "add_stats_to_use" - - private val defaultNumSnapshotPartitions: Int = 50 - - /** Canonicalize the paths for Actions */ - private[delta] def canonicalizePath(path: String, hadoopConf: Configuration): String = { - val hadoopPath = new Path(new URI(path)) - if (hadoopPath.isAbsoluteAndSchemeAuthorityNull) { - // scalastyle:off FileSystemGet - val fs = FileSystem.get(hadoopConf) - // scalastyle:on FileSystemGet - fs.makeQualified(hadoopPath).toUri.toString - } else { - // return untouched if it is a relative path or is already fully qualified - hadoopPath.toUri.toString - } - } - - /** Verifies that a set of delta or checkpoint files to be read actually belongs to this table. */ - private def assertLogFilesBelongToTable(logBasePath: Path, files: Seq[FileStatus]): Unit = { - files.map(_.getPath).foreach { - filePath => - if (new Path(filePath.toUri).getParent != new Path(logBasePath.toUri)) { - // scalastyle:off throwerror - throw new AssertionError( - s"File ($filePath) doesn't belong in the " + - s"transaction log at $logBasePath. Please contact Databricks Support.") - // scalastyle:on throwerror - } - } - } - - /** - * Metrics and metadata computed around the Delta table - * @param protocol - * The protocol version of the Delta table - * @param metadata - * The metadata of the table - * @param setTransactions - * The streaming queries writing to this table - * @param sizeInBytes - * The total size of the table (of active files, not including tombstones) - * @param numOfFiles - * The number of files in this table - * @param numOfMetadata - * The number of metadata actions in the state. Should be 1 - * @param numOfProtocol - * The number of protocol actions in the state. Should be 1 - * @param numOfRemoves - * The number of tombstones in the state - * @param numOfSetTransactions - * Number of streams writing to this table - */ - case class State( - protocol: Protocol, - metadata: Metadata, - setTransactions: Seq[SetTransaction], - sizeInBytes: Long, - numOfFiles: Long, - numOfMetadata: Long, - numOfProtocol: Long, - numOfRemoves: Long, - numOfSetTransactions: Long, - fileSizeHistogram: Option[FileSizeHistogram]) - - private[this] lazy val _stateEncoder: ExpressionEncoder[State] = - try { - ExpressionEncoder[State]() - } catch { - case e: Throwable => - logError(e.getMessage, e) - throw e - } - - implicit private def stateEncoder: Encoder[State] = { - _stateEncoder.copy() - } -} - -/** - * An initial snapshot with only metadata specified. Useful for creating a DataFrame from an - * existing parquet table during its conversion to delta. - * - * @param logPath - * the path to transaction log - * @param deltaLog - * the delta log object - * @param metadata - * the metadata of the table - */ -class InitialSnapshot( - val logPath: Path, - override val deltaLog: DeltaLog, - override val metadata: Metadata) - extends Snapshot( - path = logPath, - version = -1, - logSegment = LogSegment.empty(logPath), - minFileRetentionTimestamp = -1, - deltaLog = deltaLog, - timestamp = -1, - checksumOpt = None, - minSetTransactionRetentionTimestamp = None - ) { - - def this(logPath: Path, deltaLog: DeltaLog) = this( - logPath, - deltaLog, - Metadata( - configuration = - DeltaConfigs.mergeGlobalConfigs(SparkSession.active.sessionState.conf, Map.empty), - createdTime = Some(System.currentTimeMillis())) - ) - - override def stateDS: Dataset[SingleAction] = emptyDF.as[SingleAction] - override def stateDF: DataFrame = emptyDF - override protected lazy val computedState: Snapshot.State = initialState - private def initialState: Snapshot.State = { - val protocol = Protocol.forNewTable(spark, metadata) - Snapshot.State(protocol, metadata, Nil, 0L, 0L, 1L, 1L, 0L, 0L, None) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala deleted file mode 100644 index 29caf77b631..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/catalog/ClickHouseTableV2.scala +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.catalog - -import org.apache.spark.Partition -import org.apache.spark.internal.Logging -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.expressions.{Attribute, Expression} -import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder} -import org.apache.spark.sql.delta.{ClickhouseSnapshot, DeltaLog, DeltaTimeTravelSpec, Snapshot} -import org.apache.spark.sql.delta.actions.Metadata -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2.deltaLog2Table -import org.apache.spark.sql.delta.sources.DeltaDataSource -import org.apache.spark.sql.execution.datasources.{HadoopFsRelation, PartitionDirectory} -import org.apache.spark.sql.execution.datasources.clickhouse.utils.MergeTreePartsPartitionsUtil -import org.apache.spark.sql.execution.datasources.mergetree.StorageMeta -import org.apache.spark.sql.execution.datasources.v2.clickhouse.source.DeltaMergeTreeFileFormat -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.CaseInsensitiveStringMap -import org.apache.spark.util.collection.BitSet - -import org.apache.hadoop.fs.Path - -import java.{util => ju} - -import scala.collection.JavaConverters._ - -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) -class ClickHouseTableV2( - override val spark: SparkSession, - override val path: Path, - override val catalogTable: Option[CatalogTable] = None, - override val tableIdentifier: Option[String] = None, - override val timeTravelOpt: Option[DeltaTimeTravelSpec] = None, - override val options: Map[String, String] = Map.empty, - override val cdcOptions: CaseInsensitiveStringMap = CaseInsensitiveStringMap.empty(), - val clickhouseExtensionOptions: Map[String, String] = Map.empty) - extends DeltaTableV2( - spark, - path, - catalogTable, - tableIdentifier, - timeTravelOpt, - options, - cdcOptions) - with ClickHouseTableV2Base { - - lazy val (rootPath, partitionFilters, timeTravelByPath) = { - if (catalogTable.isDefined) { - // Fast path for reducing path munging overhead - (new Path(catalogTable.get.location), Nil, None) - } else { - DeltaDataSource.parsePathIdentifier(spark, path.toString, options) - } - } - - override protected lazy val tableSchema: StructType = schema() - - override def name(): String = - catalogTable - .map(_.identifier.unquotedString) - .orElse(tableIdentifier) - .getOrElse(s"clickhouse.`${deltaLog.dataPath}`") - - override def properties(): ju.Map[String, String] = { - val ret = super.properties() - - // for file path based write - if (snapshot.version < 0 && clickhouseExtensionOptions.nonEmpty) { - ret.putAll(clickhouseExtensionOptions.asJava) - } - ret - } - - override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { - new WriteIntoDeltaBuilder(deltaLog, info.options) - } - - def getFileFormat(meta: Metadata): DeltaMergeTreeFileFormat = { - new DeltaMergeTreeFileFormat( - StorageMeta - .withStorageID(meta, dataBaseName, tableName, ClickhouseSnapshot.genSnapshotId(snapshot))) - } - - override def deltaProperties: Map[String, String] = properties().asScala.toMap - - override def deltaCatalog: Option[CatalogTable] = catalogTable - - override def deltaPath: Path = path - - override def deltaSnapshot: Snapshot = snapshot - - def cacheThis(): Unit = { - deltaLog2Table.put(deltaLog, this) - } - - cacheThis() -} - -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) -class TempClickHouseTableV2( - override val spark: SparkSession, - override val catalogTable: Option[CatalogTable] = None) - extends ClickHouseTableV2(spark, null, catalogTable) { - import collection.JavaConverters._ - override def properties(): ju.Map[String, String] = catalogTable.get.properties.asJava - override protected def rawPartitionColumns: Seq[String] = catalogTable.get.partitionColumnNames - override def cacheThis(): Unit = {} -} - -object ClickHouseTableV2 extends Logging { - private val deltaLog2Table = - new scala.collection.concurrent.TrieMap[DeltaLog, ClickHouseTableV2]() - // for CTAS use - val temporalThreadLocalCHTable = new ThreadLocal[ClickHouseTableV2]() - - def getTable(deltaLog: DeltaLog): ClickHouseTableV2 = { - if (deltaLog2Table.contains(deltaLog)) { - deltaLog2Table(deltaLog) - } else if (temporalThreadLocalCHTable.get() != null) { - temporalThreadLocalCHTable.get() - } else { - throw new IllegalStateException( - s"Can not find ClickHouseTableV2 for deltalog ${deltaLog.dataPath}") - } - } - - def clearCache(): Unit = { - deltaLog2Table.clear() - temporalThreadLocalCHTable.remove() - } - - def partsPartitions( - deltaLog: DeltaLog, - relation: HadoopFsRelation, - selectedPartitions: Array[PartitionDirectory], - output: Seq[Attribute], - bucketedScan: Boolean, - optionalBucketSet: Option[BitSet], - optionalNumCoalescedBuckets: Option[Int], - disableBucketedScan: Boolean, - filterExprs: Seq[Expression]): Seq[Partition] = { - val tableV2 = ClickHouseTableV2.getTable(deltaLog) - - MergeTreePartsPartitionsUtil.getMergeTreePartsPartitions( - relation, - selectedPartitions, - output, - bucketedScan, - tableV2.spark, - tableV2, - optionalBucketSet, - optionalNumCoalescedBuckets, - disableBucketedScan, - filterExprs) - - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala deleted file mode 100644 index 9feac0e0b4c..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/DeleteCommand.scala +++ /dev/null @@ -1,472 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.spark.SparkContext -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.analysis.EliminateSubqueryAliases -import org.apache.spark.sql.catalyst.expressions.{EqualNullSafe, Expression, If, Literal, Not} -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.catalyst.plans.logical.{DeltaDelete, LogicalPlan} -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{Action, AddCDCFile, FileAction} -import org.apache.spark.sql.delta.commands.DeleteCommand.{rewritingFilesMsg, FINDING_TOUCHED_FILES_MSG} -import org.apache.spark.sql.delta.commands.MergeIntoCommand.totalBytesAndDistinctPartitionValues -import org.apache.spark.sql.delta.files.TahoeBatchFileIndex -import org.apache.spark.sql.execution.SQLExecution -import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric -import org.apache.spark.sql.functions.{input_file_name, lit, typedLit, udf} - -import com.fasterxml.jackson.databind.annotation.JsonDeserialize - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.2.0. It is modified to overcome the following issues: - */ - -trait DeleteCommandMetrics { self: LeafRunnableCommand => - @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() - - def createMetrics: Map[String, SQLMetric] = Map[String, SQLMetric]( - "numRemovedFiles" -> createMetric(sc, "number of files removed."), - "numAddedFiles" -> createMetric(sc, "number of files added."), - "numDeletedRows" -> createMetric(sc, "number of rows deleted."), - "numFilesBeforeSkipping" -> createMetric(sc, "number of files before skipping"), - "numBytesBeforeSkipping" -> createMetric(sc, "number of bytes before skipping"), - "numFilesAfterSkipping" -> createMetric(sc, "number of files after skipping"), - "numBytesAfterSkipping" -> createMetric(sc, "number of bytes after skipping"), - "numPartitionsAfterSkipping" -> createMetric(sc, "number of partitions after skipping"), - "numPartitionsAddedTo" -> createMetric(sc, "number of partitions added"), - "numPartitionsRemovedFrom" -> createMetric(sc, "number of partitions removed"), - "numCopiedRows" -> createMetric(sc, "number of rows copied"), - "numBytesAdded" -> createMetric(sc, "number of bytes added"), - "numBytesRemoved" -> createMetric(sc, "number of bytes removed"), - "executionTimeMs" -> createMetric(sc, "time taken to execute the entire operation"), - "scanTimeMs" -> createMetric(sc, "time taken to scan the files for matches"), - "rewriteTimeMs" -> createMetric(sc, "time taken to rewrite the matched files"), - "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), - "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), - "numTouchedRows" -> createMetric(sc, "number of rows touched") - ) -} - -/** - * Performs a Delete based on the search condition - * - * Algorithm: 1) Scan all the files and determine which files have the rows that need to be deleted. - * 2) Traverse the affected files and rebuild the touched files. 3) Use the Delta protocol to - * atomically write the remaining rows to new files and remove the affected files that are - * identified in step 1. - */ -case class DeleteCommand(deltaLog: DeltaLog, target: LogicalPlan, condition: Option[Expression]) - extends LeafRunnableCommand - with DeltaCommand - with DeleteCommandMetrics { - - override def innerChildren: Seq[QueryPlan[_]] = Seq(target) - - override lazy val metrics = createMetrics - - final override def run(sparkSession: SparkSession): Seq[Row] = { - recordDeltaOperation(deltaLog, "delta.dml.delete") { - deltaLog.assertRemovable() - deltaLog.withNewTransaction { - txn => - val deleteActions = performDelete(sparkSession, deltaLog, txn) - if (deleteActions.nonEmpty) { - txn.commit(deleteActions, DeltaOperations.Delete(condition.map(_.sql).toSeq)) - } - } - // Re-cache all cached plans(including this relation itself, if it's cached) that refer to - // this data source relation. - sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) - } - - Seq.empty[Row] - } - - def performDelete( - sparkSession: SparkSession, - deltaLog: DeltaLog, - txn: OptimisticTransaction): Seq[Action] = { - import sparkSession.implicits._ - - var numRemovedFiles: Long = 0 - var numAddedFiles: Long = 0 - var numAddedChangeFiles: Long = 0 - var scanTimeMs: Long = 0 - var rewriteTimeMs: Long = 0 - var numBytesAdded: Long = 0 - var changeFileBytes: Long = 0 - var numBytesRemoved: Long = 0 - var numFilesBeforeSkipping: Long = 0 - var numBytesBeforeSkipping: Long = 0 - var numFilesAfterSkipping: Long = 0 - var numBytesAfterSkipping: Long = 0 - var numPartitionsAfterSkipping: Option[Long] = None - var numPartitionsRemovedFrom: Option[Long] = None - var numPartitionsAddedTo: Option[Long] = None - var numDeletedRows: Option[Long] = None - var numCopiedRows: Option[Long] = None - - val startTime = System.nanoTime() - val numFilesTotal = deltaLog.snapshot.numOfFiles - - val deleteActions: Seq[Action] = condition match { - case None => - // Case 1: Delete the whole table if the condition is true - val allFiles = txn.filterFiles(Nil) - - numRemovedFiles = allFiles.size - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - val (numBytes, numPartitions) = totalBytesAndDistinctPartitionValues(allFiles) - numBytesRemoved = numBytes - numFilesBeforeSkipping = numRemovedFiles - numBytesBeforeSkipping = numBytes - numFilesAfterSkipping = numRemovedFiles - numBytesAfterSkipping = numBytes - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsAfterSkipping = Some(numPartitions) - numPartitionsRemovedFrom = Some(numPartitions) - numPartitionsAddedTo = Some(0) - } - val operationTimestamp = System.currentTimeMillis() - allFiles.map(_.removeWithTimestamp(operationTimestamp)) - case Some(cond) => - val (metadataPredicates, otherPredicates) = - DeltaTableUtils.splitMetadataAndDataPredicates( - cond, - txn.metadata.partitionColumns, - sparkSession) - - numFilesBeforeSkipping = txn.snapshot.numOfFiles - numBytesBeforeSkipping = txn.snapshot.sizeInBytes - - if (otherPredicates.isEmpty) { - // Case 2: The condition can be evaluated using metadata only. - // Delete a set of files without the need of scanning any data files. - val operationTimestamp = System.currentTimeMillis() - val candidateFiles = txn.filterFiles(metadataPredicates) - - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - numRemovedFiles = candidateFiles.size - numBytesRemoved = candidateFiles.map(_.size).sum - numFilesAfterSkipping = candidateFiles.size - val (numCandidateBytes, numCandidatePartitions) = - totalBytesAndDistinctPartitionValues(candidateFiles) - numBytesAfterSkipping = numCandidateBytes - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsAfterSkipping = Some(numCandidatePartitions) - numPartitionsRemovedFrom = Some(numCandidatePartitions) - numPartitionsAddedTo = Some(0) - } - candidateFiles.map(_.removeWithTimestamp(operationTimestamp)) - } else { - // Case 3: Delete the rows based on the condition. - val candidateFiles = txn.filterFiles(metadataPredicates ++ otherPredicates) - - numFilesAfterSkipping = candidateFiles.size - val (numCandidateBytes, numCandidatePartitions) = - totalBytesAndDistinctPartitionValues(candidateFiles) - numBytesAfterSkipping = numCandidateBytes - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsAfterSkipping = Some(numCandidatePartitions) - } - - val nameToAddFileMap = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) - - val fileIndex = new TahoeBatchFileIndex( - sparkSession, - "delete", - candidateFiles, - deltaLog, - deltaLog.dataPath, - txn.snapshot) - // Keep everything from the resolved target except a new TahoeFileIndex - // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) - val data = Dataset.ofRows(sparkSession, newTarget) - val deletedRowCount = metrics("numDeletedRows") - val deletedRowUdf = udf { - () => - deletedRowCount += 1 - true - }.asNondeterministic() - val filesToRewrite = - withStatusCode("DELTA", FINDING_TOUCHED_FILES_MSG) { - if (candidateFiles.isEmpty) { - Array.empty[String] - } else { - // --- modified start - data - .filter(new Column(cond)) - .select(input_file_name().as("input_files")) - .filter(deletedRowUdf()) - .distinct() - .as[String] - .collect() - // --- modified end - } - } - - numRemovedFiles = filesToRewrite.length - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - if (filesToRewrite.isEmpty) { - // Case 3.1: no row matches and no delete will be triggered - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsRemovedFrom = Some(0) - numPartitionsAddedTo = Some(0) - } - Nil - } else { - // Case 3.2: some files need an update to remove the deleted files - // Do the second pass and just read the affected files - val baseRelation = buildBaseRelation( - sparkSession, - txn, - "delete", - deltaLog.dataPath, - filesToRewrite, - nameToAddFileMap) - // Keep everything from the resolved target except a new TahoeFileIndex - // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) - val targetDF = Dataset.ofRows(sparkSession, newTarget) - val filterCond = Not(EqualNullSafe(cond, Literal.TrueLiteral)) - val rewrittenActions = rewriteFiles(txn, targetDF, filterCond, filesToRewrite.length) - val (changeFiles, rewrittenFiles) = rewrittenActions - .partition(_.isInstanceOf[AddCDCFile]) - numAddedFiles = rewrittenFiles.size - val removedFiles = - filesToRewrite.map(f => getTouchedFile(deltaLog.dataPath, f, nameToAddFileMap)) - val (removedBytes, removedPartitions) = - totalBytesAndDistinctPartitionValues(removedFiles) - numBytesRemoved = removedBytes - val (rewrittenBytes, rewrittenPartitions) = - totalBytesAndDistinctPartitionValues(rewrittenFiles) - numBytesAdded = rewrittenBytes - if (txn.metadata.partitionColumns.nonEmpty) { - numPartitionsRemovedFrom = Some(removedPartitions) - numPartitionsAddedTo = Some(rewrittenPartitions) - } - numAddedChangeFiles = changeFiles.size - changeFileBytes = changeFiles.collect { case f: AddCDCFile => f.size }.sum - rewriteTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - scanTimeMs - numDeletedRows = Some(metrics("numDeletedRows").value) - numCopiedRows = Some(metrics("numTouchedRows").value - metrics("numDeletedRows").value) - - val operationTimestamp = System.currentTimeMillis() - removeFilesFromPaths(deltaLog, nameToAddFileMap, filesToRewrite, operationTimestamp) ++ - rewrittenActions - } - } - } - metrics("numRemovedFiles").set(numRemovedFiles) - metrics("numAddedFiles").set(numAddedFiles) - val executionTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - metrics("executionTimeMs").set(executionTimeMs) - metrics("scanTimeMs").set(scanTimeMs) - metrics("rewriteTimeMs").set(rewriteTimeMs) - metrics("numAddedChangeFiles").set(numAddedChangeFiles) - metrics("changeFileBytes").set(changeFileBytes) - metrics("numBytesAdded").set(numBytesAdded) - metrics("numBytesRemoved").set(numBytesRemoved) - metrics("numFilesBeforeSkipping").set(numFilesBeforeSkipping) - metrics("numBytesBeforeSkipping").set(numBytesBeforeSkipping) - metrics("numFilesAfterSkipping").set(numFilesAfterSkipping) - metrics("numBytesAfterSkipping").set(numBytesAfterSkipping) - numPartitionsAfterSkipping.foreach(metrics("numPartitionsAfterSkipping").set) - numPartitionsAddedTo.foreach(metrics("numPartitionsAddedTo").set) - numPartitionsRemovedFrom.foreach(metrics("numPartitionsRemovedFrom").set) - numCopiedRows.foreach(metrics("numCopiedRows").set) - txn.registerSQLMetrics(sparkSession, metrics) - // This is needed to make the SQL metrics visible in the Spark UI - val executionId = sparkSession.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates(sparkSession.sparkContext, executionId, metrics.values.toSeq) - - recordDeltaEvent( - deltaLog, - "delta.dml.delete.stats", - data = DeleteMetric( - condition = condition.map(_.sql).getOrElse("true"), - numFilesTotal, - numFilesAfterSkipping, - numAddedFiles, - numRemovedFiles, - numAddedFiles, - numAddedChangeFiles = numAddedChangeFiles, - numFilesBeforeSkipping, - numBytesBeforeSkipping, - numFilesAfterSkipping, - numBytesAfterSkipping, - numPartitionsAfterSkipping, - numPartitionsAddedTo, - numPartitionsRemovedFrom, - numCopiedRows, - numDeletedRows, - numBytesAdded, - numBytesRemoved, - changeFileBytes = changeFileBytes, - scanTimeMs, - rewriteTimeMs - ) - ) - - deleteActions - } - - /** Returns the list of [[AddFile]]s and [[AddCDCFile]]s that have been re-written. */ - private def rewriteFiles( - txn: OptimisticTransaction, - baseData: DataFrame, - filterCondition: Expression, - numFilesToRewrite: Long): Seq[FileAction] = { - val shouldWriteCdc = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) - - // number of total rows that we have seen / are either copying or deleting (sum of both). - val numTouchedRows = metrics("numTouchedRows") - val numTouchedRowsUdf = udf { - () => - numTouchedRows += 1 - true - }.asNondeterministic() - - withStatusCode("DELTA", rewritingFilesMsg(numFilesToRewrite)) { - val dfToWrite = if (shouldWriteCdc) { - import org.apache.spark.sql.delta.commands.cdc.CDCReader._ - // The logic here ends up being surprisingly elegant, with all source rows ending up in - // the output. Recall that we flipped the user-provided delete condition earlier, before the - // call to `rewriteFiles`. All rows which match this latest `filterCondition` are retained - // as table data, while all rows which don't match are removed from the rewritten table data - // but do get included in the output as CDC events. - baseData - .filter(numTouchedRowsUdf()) - .withColumn( - CDC_TYPE_COLUMN_NAME, - new Column( - If( - filterCondition, - typedLit[String](CDC_TYPE_NOT_CDC).expr, - lit(CDC_TYPE_DELETE).expr) - ) - ) - } else { - baseData - .filter(numTouchedRowsUdf()) - .filter(new Column(filterCondition)) - } - - txn.writeFiles(dfToWrite) - } - } -} - -object DeleteCommand { - def apply(delete: DeltaDelete): DeleteCommand = { - val index = EliminateSubqueryAliases(delete.child) match { - case DeltaFullTable(tahoeFileIndex) => - tahoeFileIndex - case o => - throw DeltaErrors.notADeltaSourceException("DELETE", Some(o)) - } - DeleteCommand(index.deltaLog, delete.child, delete.condition) - } - - val FILE_NAME_COLUMN: String = "_input_file_name_" - val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for DELETE operation" - - def rewritingFilesMsg(numFilesToRewrite: Long): String = - s"Rewriting $numFilesToRewrite files for DELETE operation" -} - -/** - * Used to report details about delete. - * - * @param condition: - * what was the delete condition - * @param numFilesTotal: - * how big is the table - * @param numTouchedFiles: - * how many files did we touch. Alias for `numFilesAfterSkipping` - * @param numRewrittenFiles: - * how many files had to be rewritten. Alias for `numAddedFiles` - * @param numRemovedFiles: - * how many files we removed. Alias for `numTouchedFiles` - * @param numAddedFiles: - * how many files we added. Alias for `numRewrittenFiles` - * @param numAddedChangeFiles: - * how many change files were generated - * @param numFilesBeforeSkipping: - * how many candidate files before skipping - * @param numBytesBeforeSkipping: - * how many candidate bytes before skipping - * @param numFilesAfterSkipping: - * how many candidate files after skipping - * @param numBytesAfterSkipping: - * how many candidate bytes after skipping - * @param numPartitionsAfterSkipping: - * how many candidate partitions after skipping - * @param numPartitionsAddedTo: - * how many new partitions were added - * @param numPartitionsRemovedFrom: - * how many partitions were removed - * @param numCopiedRows: - * how many rows were copied - * @param numDeletedRows: - * how many rows were deleted - * @param numBytesAdded: - * how many bytes were added - * @param numBytesRemoved: - * how many bytes were removed - * @param changeFileBytes: - * total size of change files generated - * @param scanTimeMs: - * how long did finding take - * @param rewriteTimeMs: - * how long did rewriting take - * - * @note - * All the time units are milliseconds. - */ -case class DeleteMetric( - condition: String, - numFilesTotal: Long, - numTouchedFiles: Long, - numRewrittenFiles: Long, - numRemovedFiles: Long, - numAddedFiles: Long, - numAddedChangeFiles: Long, - numFilesBeforeSkipping: Long, - numBytesBeforeSkipping: Long, - numFilesAfterSkipping: Long, - numBytesAfterSkipping: Long, - numPartitionsAfterSkipping: Option[Long], - numPartitionsAddedTo: Option[Long], - numPartitionsRemovedFrom: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - numCopiedRows: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - numDeletedRows: Option[Long], - numBytesAdded: Long, - numBytesRemoved: Long, - changeFileBytes: Long, - scanTimeMs: Long, - rewriteTimeMs: Long -) diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala deleted file mode 100644 index 4882def7b2e..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/MergeIntoCommand.scala +++ /dev/null @@ -1,1135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.spark.SparkContext -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.encoders.{ExpressionEncoder, RowEncoder} -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeReference, BasePredicate, Expression, Literal, NamedExpression, PredicateHelper, UnsafeProjection} -import org.apache.spark.sql.catalyst.expressions.codegen._ -import org.apache.spark.sql.catalyst.plans.logical._ -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.files._ -import org.apache.spark.sql.delta.schema.{ImplicitMetadataOperation, SchemaUtils} -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.{AnalysisHelper, SetAccumulator} -import org.apache.spark.sql.execution.SQLExecution -import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.datasources.LogicalRelation -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.{DataTypes, StructType} - -import com.fasterxml.jackson.databind.annotation.JsonDeserialize - -import java.util.concurrent.TimeUnit - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.2.0. It is modified to overcome the following issues: - */ - -case class MergeDataSizes( - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - rows: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - files: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - bytes: Option[Long] = None, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - partitions: Option[Long] = None) - -/** - * Represents the state of a single merge clause: - * - merge clause's (optional) predicate - * - action type (insert, update, delete) - * - action's expressions - */ -case class MergeClauseStats(condition: Option[String], actionType: String, actionExpr: Seq[String]) - -object MergeClauseStats { - def apply(mergeClause: DeltaMergeIntoClause): MergeClauseStats = { - MergeClauseStats( - condition = mergeClause.condition.map(_.sql), - mergeClause.clauseType.toLowerCase(), - actionExpr = mergeClause.actions.map(_.sql)) - } -} - -/** State for a merge operation */ -case class MergeStats( - // Merge condition expression - conditionExpr: String, - - // Expressions used in old MERGE stats, now always Null - updateConditionExpr: String, - updateExprs: Seq[String], - insertConditionExpr: String, - insertExprs: Seq[String], - deleteConditionExpr: String, - - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED - matchedStats: Seq[MergeClauseStats], - notMatchedStats: Seq[MergeClauseStats], - - // Data sizes of source and target at different stages of processing - source: MergeDataSizes, - targetBeforeSkipping: MergeDataSizes, - targetAfterSkipping: MergeDataSizes, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - sourceRowsInSecondScan: Option[Long], - - // Data change sizes - targetFilesRemoved: Long, - targetFilesAdded: Long, - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetChangeFilesAdded: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetChangeFileBytes: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetBytesRemoved: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetBytesAdded: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetPartitionsRemovedFrom: Option[Long], - @JsonDeserialize(contentAs = classOf[java.lang.Long]) - targetPartitionsAddedTo: Option[Long], - targetRowsCopied: Long, - targetRowsUpdated: Long, - targetRowsInserted: Long, - targetRowsDeleted: Long -) - -object MergeStats { - - def fromMergeSQLMetrics( - metrics: Map[String, SQLMetric], - condition: Expression, - matchedClauses: Seq[DeltaMergeIntoMatchedClause], - notMatchedClauses: Seq[DeltaMergeIntoInsertClause], - isPartitioned: Boolean): MergeStats = { - - def metricValueIfPartitioned(metricName: String): Option[Long] = { - if (isPartitioned) Some(metrics(metricName).value) else None - } - - MergeStats( - // Merge condition expression - conditionExpr = condition.sql, - - // Newer expressions used in MERGE with any number of MATCHED/NOT MATCHED - matchedStats = matchedClauses.map(MergeClauseStats(_)), - notMatchedStats = notMatchedClauses.map(MergeClauseStats(_)), - - // Data sizes of source and target at different stages of processing - source = MergeDataSizes(rows = Some(metrics("numSourceRows").value)), - targetBeforeSkipping = MergeDataSizes( - files = Some(metrics("numTargetFilesBeforeSkipping").value), - bytes = Some(metrics("numTargetBytesBeforeSkipping").value)), - targetAfterSkipping = MergeDataSizes( - files = Some(metrics("numTargetFilesAfterSkipping").value), - bytes = Some(metrics("numTargetBytesAfterSkipping").value), - partitions = metricValueIfPartitioned("numTargetPartitionsAfterSkipping") - ), - sourceRowsInSecondScan = metrics.get("numSourceRowsInSecondScan").map(_.value).filter(_ >= 0), - - // Data change sizes - targetFilesAdded = metrics("numTargetFilesAdded").value, - targetChangeFilesAdded = metrics.get("numTargetChangeFilesAdded").map(_.value), - targetChangeFileBytes = metrics.get("numTargetChangeFileBytes").map(_.value), - targetFilesRemoved = metrics("numTargetFilesRemoved").value, - targetBytesAdded = Some(metrics("numTargetBytesAdded").value), - targetBytesRemoved = Some(metrics("numTargetBytesRemoved").value), - targetPartitionsRemovedFrom = metricValueIfPartitioned("numTargetPartitionsRemovedFrom"), - targetPartitionsAddedTo = metricValueIfPartitioned("numTargetPartitionsAddedTo"), - targetRowsCopied = metrics("numTargetRowsCopied").value, - targetRowsUpdated = metrics("numTargetRowsUpdated").value, - targetRowsInserted = metrics("numTargetRowsInserted").value, - targetRowsDeleted = metrics("numTargetRowsDeleted").value, - - // Deprecated fields - updateConditionExpr = null, - updateExprs = null, - insertConditionExpr = null, - insertExprs = null, - deleteConditionExpr = null - ) - } -} - -/** - * Performs a merge of a source query/table into a Delta table. - * - * Issues an error message when the ON search_condition of the MERGE statement can match a single - * row from the target table with multiple rows of the source table-reference. - * - * Algorithm: - * - * Phase 1: Find the input files in target that are touched by the rows that satisfy the condition - * and verify that no two source rows match with the same target row. This is implemented as an - * inner-join using the given condition. See [[findTouchedFiles]] for more details. - * - * Phase 2: Read the touched files again and write new files with updated and/or inserted rows. - * - * Phase 3: Use the Delta protocol to atomically remove the touched files and add the new files. - * - * @param source - * Source data to merge from - * @param target - * Target table to merge into - * @param targetFileIndex - * TahoeFileIndex of the target table - * @param condition - * Condition for a source row to match with a target row - * @param matchedClauses - * All info related to matched clauses. - * @param notMatchedClauses - * All info related to not matched clause. - * @param migratedSchema - * The final schema of the target - may be changed by schema evolution. - */ -case class MergeIntoCommand( - @transient source: LogicalPlan, - @transient target: LogicalPlan, - @transient targetFileIndex: TahoeFileIndex, - condition: Expression, - matchedClauses: Seq[DeltaMergeIntoMatchedClause], - notMatchedClauses: Seq[DeltaMergeIntoInsertClause], - migratedSchema: Option[StructType]) - extends LeafRunnableCommand - with DeltaCommand - with PredicateHelper - with AnalysisHelper - with ImplicitMetadataOperation { - - import org.apache.spark.sql.delta.commands.cdc.CDCReader._ - - import MergeIntoCommand._ - import SQLMetrics._ - - override val canMergeSchema: Boolean = conf.getConf(DeltaSQLConf.DELTA_SCHEMA_AUTO_MIGRATE) - override val canOverwriteSchema: Boolean = false - - @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() - @transient private lazy val targetDeltaLog: DeltaLog = targetFileIndex.deltaLog - - /** - * Map to get target output attributes by name. The case sensitivity of the map is set accordingly - * to Spark configuration. - */ - @transient private lazy val targetOutputAttributesMap: Map[String, Attribute] = { - val attrMap: Map[String, Attribute] = target.outputSet.view - .map(attr => attr.name -> attr) - .toMap - if (conf.caseSensitiveAnalysis) { - attrMap - } else { - CaseInsensitiveMap(attrMap) - } - } - - /** Whether this merge statement has only a single insert (NOT MATCHED) clause. */ - private def isSingleInsertOnly: Boolean = matchedClauses.isEmpty && notMatchedClauses.length == 1 - - /** Whether this merge statement has only MATCHED clauses. */ - private def isMatchedOnly: Boolean = notMatchedClauses.isEmpty && matchedClauses.nonEmpty - - // We over-count numTargetRowsDeleted when there are multiple matches; - // this is the amount of the overcount, so we can subtract it to get a correct final metric. - private var multipleMatchDeleteOnlyOvercount: Option[Long] = None - - override lazy val metrics = Map[String, SQLMetric]( - "numSourceRows" -> createMetric(sc, "number of source rows"), - "numSourceRowsInSecondScan" -> - createMetric(sc, "number of source rows (during repeated scan)"), - "numTargetRowsCopied" -> createMetric(sc, "number of target rows rewritten unmodified"), - "numTargetRowsInserted" -> createMetric(sc, "number of inserted rows"), - "numTargetRowsUpdated" -> createMetric(sc, "number of updated rows"), - "numTargetRowsDeleted" -> createMetric(sc, "number of deleted rows"), - "numTargetFilesBeforeSkipping" -> createMetric(sc, "number of target files before skipping"), - "numTargetFilesAfterSkipping" -> createMetric(sc, "number of target files after skipping"), - "numTargetFilesRemoved" -> createMetric(sc, "number of files removed to target"), - "numTargetFilesAdded" -> createMetric(sc, "number of files added to target"), - "numTargetChangeFilesAdded" -> - createMetric(sc, "number of change data capture files generated"), - "numTargetChangeFileBytes" -> - createMetric(sc, "total size of change data capture files generated"), - "numTargetBytesBeforeSkipping" -> createMetric(sc, "number of target bytes before skipping"), - "numTargetBytesAfterSkipping" -> createMetric(sc, "number of target bytes after skipping"), - "numTargetBytesRemoved" -> createMetric(sc, "number of target bytes removed"), - "numTargetBytesAdded" -> createMetric(sc, "number of target bytes added"), - "numTargetPartitionsAfterSkipping" -> - createMetric(sc, "number of target partitions after skipping"), - "numTargetPartitionsRemovedFrom" -> - createMetric(sc, "number of target partitions from which files were removed"), - "numTargetPartitionsAddedTo" -> - createMetric(sc, "number of target partitions to which files were added"), - "executionTimeMs" -> - createMetric(sc, "time taken to execute the entire operation"), - "scanTimeMs" -> - createMetric(sc, "time taken to scan the files for matches"), - "rewriteTimeMs" -> - createMetric(sc, "time taken to rewrite the matched files") - ) - - override def run(spark: SparkSession): Seq[Row] = { - if (migratedSchema.isDefined) { - // Block writes of void columns in the Delta log. Currently void columns are not properly - // supported and are dropped on read, but this is not enough for merge command that is also - // reading the schema from the Delta log. Until proper support we prefer to fail merge - // queries that add void columns. - val newNullColumn = SchemaUtils.findNullTypeColumn(migratedSchema.get) - if (newNullColumn.isDefined) { - throw new AnalysisException( - s"""Cannot add column '${newNullColumn.get}' with type 'void'. Please explicitly specify a - |non-void type.""".stripMargin.replaceAll("\n", " ") - ) - } - } - - recordDeltaOperation(targetDeltaLog, "delta.dml.merge") { - val startTime = System.nanoTime() - targetDeltaLog.withNewTransaction { - deltaTxn => - if (target.schema.size != deltaTxn.metadata.schema.size) { - throw DeltaErrors.schemaChangedSinceAnalysis( - atAnalysis = target.schema, - latestSchema = deltaTxn.metadata.schema) - } - - if (canMergeSchema) { - updateMetadata( - spark, - deltaTxn, - migratedSchema.getOrElse(target.schema), - deltaTxn.metadata.partitionColumns, - deltaTxn.metadata.configuration, - isOverwriteMode = false, - rearrangeOnly = false - ) - } - - val deltaActions = { - if (isSingleInsertOnly && spark.conf.get(DeltaSQLConf.MERGE_INSERT_ONLY_ENABLED)) { - writeInsertsOnlyWhenNoMatchedClauses(spark, deltaTxn) - } else { - val filesToRewrite = findTouchedFiles(spark, deltaTxn) - val newWrittenFiles = withStatusCode("DELTA", "Writing merged data") { - writeAllChanges(spark, deltaTxn, filesToRewrite) - } - filesToRewrite.map(_.remove) ++ newWrittenFiles - } - } - - // Metrics should be recorded before commit (where they are written to delta logs). - metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) - deltaTxn.registerSQLMetrics(spark, metrics) - - // This is a best-effort sanity check. - if ( - metrics("numSourceRowsInSecondScan").value >= 0 && - metrics("numSourceRows").value != metrics("numSourceRowsInSecondScan").value - ) { - log.warn( - s"Merge source has ${metrics("numSourceRows")} rows in initial scan but " + - s"${metrics("numSourceRowsInSecondScan")} rows in second scan") - if (conf.getConf(DeltaSQLConf.MERGE_FAIL_IF_SOURCE_CHANGED)) { - throw DeltaErrors.sourceNotDeterministicInMergeException(spark) - } - } - - deltaTxn.commit( - deltaActions, - DeltaOperations.Merge( - Option(condition.sql), - matchedClauses.map(DeltaOperations.MergePredicate(_)), - notMatchedClauses.map(DeltaOperations.MergePredicate(_))) - ) - - // Record metrics - val stats = MergeStats.fromMergeSQLMetrics( - metrics, - condition, - matchedClauses, - notMatchedClauses, - deltaTxn.metadata.partitionColumns.nonEmpty) - recordDeltaEvent(targetFileIndex.deltaLog, "delta.dml.merge.stats", data = stats) - - } - spark.sharedState.cacheManager.recacheByPlan(spark, target) - } - // This is needed to make the SQL metrics visible in the Spark UI. Also this needs - // to be outside the recordMergeOperation because this method will update some metric. - val executionId = spark.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates(spark.sparkContext, executionId, metrics.values.toSeq) - Seq.empty - } - - /** - * Find the target table files that contain the rows that satisfy the merge condition. This is - * implemented as an inner-join between the source query/table and the target table using the - * merge condition. - */ - private def findTouchedFiles( - spark: SparkSession, - deltaTxn: OptimisticTransaction - ): Seq[AddFile] = recordMergeOperation(sqlMetricName = "scanTimeMs") { - - // Accumulator to collect all the distinct touched files - val touchedFilesAccum = new SetAccumulator[String]() - spark.sparkContext.register(touchedFilesAccum, TOUCHED_FILES_ACCUM_NAME) - - // UDFs to records touched files names and add them to the accumulator - val recordTouchedFileName = udf { - (fileName: String) => - { - // --- modified start - fileName.split(",").foreach(name => touchedFilesAccum.add(name)) - // --- modified end - 1 - } - }.asNondeterministic() - - // Skip data based on the merge condition - val targetOnlyPredicates = - splitConjunctivePredicates(condition).filter(_.references.subsetOf(target.outputSet)) - val dataSkippedFiles = deltaTxn.filterFiles(targetOnlyPredicates) - - // UDF to increment metrics - val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRows") - val sourceDF = Dataset - .ofRows(spark, source) - .filter(new Column(incrSourceRowCountExpr)) - - // Apply inner join to between source and target using the merge condition to find matches - // In addition, we attach two columns - // - a monotonically increasing row id for target rows to later identify whether the same - // target row is modified by multiple user or not - // - the target file name the row is from to later identify the files touched by matched rows - val targetDF = Dataset - .ofRows(spark, buildTargetPlanWithFiles(deltaTxn, dataSkippedFiles)) - .withColumn(ROW_ID_COL, monotonically_increasing_id()) - .withColumn(FILE_NAME_COL, input_file_name()) - val joinToFindTouchedFiles = sourceDF.join(targetDF, new Column(condition), "inner") - - // Process the matches from the inner join to record touched files and find multiple matches - val collectTouchedFiles = joinToFindTouchedFiles - .select(col(ROW_ID_COL), recordTouchedFileName(col(FILE_NAME_COL)).as("one")) - - // Calculate frequency of matches per source row - val matchedRowCounts = collectTouchedFiles.groupBy(ROW_ID_COL).agg(sum("one").as("count")) - - // Get multiple matches and simultaneously collect (using touchedFilesAccum) the file names - // multipleMatchCount = # of target rows with more than 1 matching source row (duplicate match) - // multipleMatchSum = total # of duplicate matched rows - import spark.implicits._ - val (multipleMatchCount, multipleMatchSum) = matchedRowCounts - .filter("count > 1") - .select(coalesce(count("*"), lit(0)), coalesce(sum("count"), lit(0))) - .as[(Long, Long)] - .collect() - .head - - val hasMultipleMatches = multipleMatchCount > 0 - - // Throw error if multiple matches are ambiguous or cannot be computed correctly. - val canBeComputedUnambiguously = { - // Multiple matches are not ambiguous when there is only one unconditional delete as - // all the matched row pairs in the 2nd join in `writeAllChanges` will get deleted. - val isUnconditionalDelete = matchedClauses.headOption match { - case Some(DeltaMergeIntoDeleteClause(None)) => true - case _ => false - } - matchedClauses.size == 1 && isUnconditionalDelete - } - - if (hasMultipleMatches && !canBeComputedUnambiguously) { - throw DeltaErrors.multipleSourceRowMatchingTargetRowInMergeException(spark) - } - - if (hasMultipleMatches) { - // This is only allowed for delete-only queries. - // This query will count the duplicates for numTargetRowsDeleted in Job 2, - // because we count matches after the join and not just the target rows. - // We have to compensate for this by subtracting the duplicates later, - // so we need to record them here. - val duplicateCount = multipleMatchSum - multipleMatchCount - multipleMatchDeleteOnlyOvercount = Some(duplicateCount) - } - - // Get the AddFiles using the touched file names. - val touchedFileNames = touchedFilesAccum.value.iterator().asScala.toSeq - logTrace(s"findTouchedFiles: matched files:\n\t${touchedFileNames.mkString("\n\t")}") - - val nameToAddFileMap = generateCandidateFileMap(targetDeltaLog.dataPath, dataSkippedFiles) - val touchedAddFiles = - touchedFileNames.map(f => getTouchedFile(targetDeltaLog.dataPath, f, nameToAddFileMap)) - - // When the target table is empty, and the optimizer optimized away the join entirely - // numSourceRows will be incorrectly 0. We need to scan the source table once to get the correct - // metric here. - if ( - metrics("numSourceRows").value == 0 && - (dataSkippedFiles.isEmpty || targetDF.take(1).isEmpty) - ) { - val numSourceRows = sourceDF.count() - metrics("numSourceRows").set(numSourceRows) - } - - // Update metrics - metrics("numTargetFilesBeforeSkipping") += deltaTxn.snapshot.numOfFiles - metrics("numTargetBytesBeforeSkipping") += deltaTxn.snapshot.sizeInBytes - val (afterSkippingBytes, afterSkippingPartitions) = - totalBytesAndDistinctPartitionValues(dataSkippedFiles) - metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size - metrics("numTargetBytesAfterSkipping") += afterSkippingBytes - metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions - val (removedBytes, removedPartitions) = totalBytesAndDistinctPartitionValues(touchedAddFiles) - metrics("numTargetFilesRemoved") += touchedAddFiles.size - metrics("numTargetBytesRemoved") += removedBytes - metrics("numTargetPartitionsRemovedFrom") += removedPartitions - touchedAddFiles - } - - /** - * This is an optimization of the case when there is no update clause for the merge. We perform an - * left anti join on the source data to find the rows to be inserted. - * - * This will currently only optimize for the case when there is a _single_ notMatchedClause. - */ - private def writeInsertsOnlyWhenNoMatchedClauses( - spark: SparkSession, - deltaTxn: OptimisticTransaction - ): Seq[FileAction] = recordMergeOperation(sqlMetricName = "rewriteTimeMs") { - - // UDFs to update metrics - val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRows") - val incrInsertedCountExpr = makeMetricUpdateUDF("numTargetRowsInserted") - - val outputColNames = getTargetOutputCols(deltaTxn).map(_.name) - // we use head here since we know there is only a single notMatchedClause - val outputExprs = notMatchedClauses.head.resolvedActions.map(_.expr) - val outputCols = outputExprs.zip(outputColNames).map { - case (expr, name) => - new Column(Alias(expr, name)()) - } - - // source DataFrame - val sourceDF = Dataset - .ofRows(spark, source) - .filter(new Column(incrSourceRowCountExpr)) - .filter(new Column(notMatchedClauses.head.condition.getOrElse(Literal.TrueLiteral))) - - // Skip data based on the merge condition - val conjunctivePredicates = splitConjunctivePredicates(condition) - val targetOnlyPredicates = - conjunctivePredicates.filter(_.references.subsetOf(target.outputSet)) - val dataSkippedFiles = deltaTxn.filterFiles(targetOnlyPredicates) - - // target DataFrame - val targetDF = Dataset.ofRows(spark, buildTargetPlanWithFiles(deltaTxn, dataSkippedFiles)) - - val insertDf = sourceDF - .join(targetDF, new Column(condition), "leftanti") - .select(outputCols: _*) - .filter(new Column(incrInsertedCountExpr)) - - val newFiles = deltaTxn - .writeFiles(repartitionIfNeeded(spark, insertDf, deltaTxn.metadata.partitionColumns)) - - // Update metrics - metrics("numTargetFilesBeforeSkipping") += deltaTxn.snapshot.numOfFiles - metrics("numTargetBytesBeforeSkipping") += deltaTxn.snapshot.sizeInBytes - val (afterSkippingBytes, afterSkippingPartitions) = - totalBytesAndDistinctPartitionValues(dataSkippedFiles) - metrics("numTargetFilesAfterSkipping") += dataSkippedFiles.size - metrics("numTargetBytesAfterSkipping") += afterSkippingBytes - metrics("numTargetPartitionsAfterSkipping") += afterSkippingPartitions - metrics("numTargetFilesRemoved") += 0 - metrics("numTargetBytesRemoved") += 0 - metrics("numTargetPartitionsRemovedFrom") += 0 - val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) - metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) - metrics("numTargetBytesAdded") += addedBytes - metrics("numTargetPartitionsAddedTo") += addedPartitions - newFiles - } - - /** - * Write new files by reading the touched files and updating/inserting data using the source - * query/table. This is implemented using a full|right-outer-join using the merge condition. - * - * Note that unlike the insert-only code paths with just one control column INCR_ROW_COUNT_COL, - * this method has two additional control columns ROW_DROPPED_COL for dropping deleted rows and - * CDC_TYPE_COL_NAME used for handling CDC when enabled. - */ - private def writeAllChanges( - spark: SparkSession, - deltaTxn: OptimisticTransaction, - filesToRewrite: Seq[AddFile] - ): Seq[FileAction] = recordMergeOperation(sqlMetricName = "rewriteTimeMs") { - import org.apache.spark.sql.catalyst.expressions.Literal.{FalseLiteral, TrueLiteral} - - val cdcEnabled = DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(deltaTxn.metadata) - - var targetOutputCols = getTargetOutputCols(deltaTxn) - var outputRowSchema = deltaTxn.metadata.schema - - // When we have duplicate matches (only allowed when the whenMatchedCondition is a delete with - // no match condition) we will incorrectly generate duplicate CDC rows. - // Duplicate matches can be due to: - // - Duplicate rows in the source w.r.t. the merge condition - // - A target-only or source-only merge condition, which essentially turns our join into a cross - // join with the target/source satisfiying the merge condition. - // These duplicate matches are dropped from the main data output since this is a delete - // operation, but the duplicate CDC rows are not removed by default. - // See https://github.com/delta-io/delta/issues/1274 - - // We address this specific scenario by adding row ids to the target before performing our join. - // There should only be one CDC delete row per target row so we can use these row ids to dedupe - // the duplicate CDC delete rows. - - // We also need to address the scenario when there are duplicate matches with delete and we - // insert duplicate rows. Here we need to additionally add row ids to the source before the - // join to avoid dropping these valid duplicate inserted rows and their corresponding cdc rows. - - // When there is an insert clause, we set SOURCE_ROW_ID_COL=null for all delete rows because we - // need to drop the duplicate matches. - val isDeleteWithDuplicateMatchesAndCdc = multipleMatchDeleteOnlyOvercount.nonEmpty && cdcEnabled - - // Generate a new logical plan that has same output attributes exprIds as the target plan. - // This allows us to apply the existing resolved update/insert expressions. - val newTarget = buildTargetPlanWithFiles(deltaTxn, filesToRewrite) - val joinType = - if ( - isMatchedOnly && - spark.conf.get(DeltaSQLConf.MERGE_MATCHED_ONLY_ENABLED) - ) { - "rightOuter" - } else { - "fullOuter" - } - - logDebug(s"""writeAllChanges using $joinType join: - | source.output: ${source.outputSet} - | target.output: ${target.outputSet} - | condition: $condition - | newTarget.output: ${newTarget.outputSet} - """.stripMargin) - - // UDFs to update metrics - val incrSourceRowCountExpr = makeMetricUpdateUDF("numSourceRowsInSecondScan") - val incrUpdatedCountExpr = makeMetricUpdateUDF("numTargetRowsUpdated") - val incrInsertedCountExpr = makeMetricUpdateUDF("numTargetRowsInserted") - val incrNoopCountExpr = makeMetricUpdateUDF("numTargetRowsCopied") - val incrDeletedCountExpr = makeMetricUpdateUDF("numTargetRowsDeleted") - - // Apply an outer join to find both, matches and non-matches. We are adding two boolean fields - // with value `true`, one to each side of the join. Whether this field is null or not after - // the outer join, will allow us to identify whether the resultant joined row was a - // matched inner result or an unmatched result with null on one side. - // We add row IDs to the targetDF if we have a delete-when-matched clause with duplicate - // matches and CDC is enabled, and additionally add row IDs to the source if we also have an - // insert clause. See above at isDeleteWithDuplicateMatchesAndCdc definition for more details. - var sourceDF = Dataset - .ofRows(spark, source) - .withColumn(SOURCE_ROW_PRESENT_COL, new Column(incrSourceRowCountExpr)) - var targetDF = Dataset - .ofRows(spark, newTarget) - .withColumn(TARGET_ROW_PRESENT_COL, lit(true)) - if (isDeleteWithDuplicateMatchesAndCdc) { - targetDF = targetDF.withColumn(TARGET_ROW_ID_COL, monotonically_increasing_id()) - if (notMatchedClauses.nonEmpty) { // insert clause - sourceDF = sourceDF.withColumn(SOURCE_ROW_ID_COL, monotonically_increasing_id()) - } - } - val joinedDF = sourceDF.join(targetDF, new Column(condition), joinType) - val joinedPlan = joinedDF.queryExecution.analyzed - - def resolveOnJoinedPlan(exprs: Seq[Expression]): Seq[Expression] = { - tryResolveReferencesForExpressions(spark, exprs, joinedPlan) - } - - // ==== Generate the expressions to process full-outer join output and generate target rows ==== - // If there are N columns in the target table, there will be N + 3 columns after processing - // - N columns for target table - // - ROW_DROPPED_COL to define whether the generated row should dropped or written - // - INCR_ROW_COUNT_COL containing a UDF to update the output row row counter - // - CDC_TYPE_COLUMN_NAME containing the type of change being performed in a particular row - - // To generate these N + 3 columns, we will generate N + 3 expressions and apply them to the - // rows in the joinedDF. The CDC column will be either used for CDC generation or dropped before - // performing the final write, and the other two will always be dropped after executing the - // metrics UDF and filtering on ROW_DROPPED_COL. - - // We produce rows for both the main table data (with CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC), - // and rows for the CDC data which will be output to CDCReader.CDC_LOCATION. - // See [[CDCReader]] for general details on how partitioning on the CDC type column works. - - // In the following two functions `matchedClauseOutput` and `notMatchedClauseOutput`, we - // produce a Seq[Expression] for each intended output row. - // Depending on the clause and whether CDC is enabled, we output between 0 and 3 rows, as a - // Seq[Seq[Expression]] - - // There is one corner case outlined above at isDeleteWithDuplicateMatchesAndCdc definition. - // When we have a delete-ONLY merge with duplicate matches we have N + 4 columns: - // N target cols, TARGET_ROW_ID_COL, ROW_DROPPED_COL, INCR_ROW_COUNT_COL, CDC_TYPE_COLUMN_NAME - // When we have a delete-when-matched merge with duplicate matches + an insert clause, we have - // N + 5 columns: - // N target cols, TARGET_ROW_ID_COL, SOURCE_ROW_ID_COL, ROW_DROPPED_COL, INCR_ROW_COUNT_COL, - // CDC_TYPE_COLUMN_NAME - // These ROW_ID_COL will always be dropped before the final write. - - if (isDeleteWithDuplicateMatchesAndCdc) { - targetOutputCols = targetOutputCols :+ UnresolvedAttribute(TARGET_ROW_ID_COL) - outputRowSchema = outputRowSchema.add(TARGET_ROW_ID_COL, DataTypes.LongType) - if (notMatchedClauses.nonEmpty) { // there is an insert clause, make SRC_ROW_ID_COL=null - targetOutputCols = targetOutputCols :+ Alias(Literal(null), SOURCE_ROW_ID_COL)() - outputRowSchema = outputRowSchema.add(SOURCE_ROW_ID_COL, DataTypes.LongType) - } - } - - if (cdcEnabled) { - outputRowSchema = outputRowSchema - .add(ROW_DROPPED_COL, DataTypes.BooleanType) - .add(INCR_ROW_COUNT_COL, DataTypes.BooleanType) - .add(CDC_TYPE_COLUMN_NAME, DataTypes.StringType) - } - - def matchedClauseOutput(clause: DeltaMergeIntoMatchedClause): Seq[Seq[Expression]] = { - val exprs = clause match { - case u: DeltaMergeIntoUpdateClause => - // Generate update expressions and set ROW_DELETED_COL = false and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC - val mainDataOutput = u.resolvedActions.map(_.expr) :+ FalseLiteral :+ - incrUpdatedCountExpr :+ Literal(CDC_TYPE_NOT_CDC) - if (cdcEnabled) { - // For update preimage, we have do a no-op copy with ROW_DELETED_COL = false and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_UPDATE_PREIMAGE and INCR_ROW_COUNT_COL as a no-op - // (because the metric will be incremented in `mainDataOutput`) - val preImageOutput = targetOutputCols :+ FalseLiteral :+ TrueLiteral :+ - Literal(CDC_TYPE_UPDATE_PREIMAGE) - // For update postimage, we have the same expressions as for mainDataOutput but with - // INCR_ROW_COUNT_COL as a no-op (because the metric will be incremented in - // `mainDataOutput`), and CDC_TYPE_COLUMN_NAME = CDC_TYPE_UPDATE_POSTIMAGE - val postImageOutput = mainDataOutput.dropRight(2) :+ TrueLiteral :+ - Literal(CDC_TYPE_UPDATE_POSTIMAGE) - Seq(mainDataOutput, preImageOutput, postImageOutput) - } else { - Seq(mainDataOutput) - } - case _: DeltaMergeIntoDeleteClause => - // Generate expressions to set the ROW_DELETED_COL = true and CDC_TYPE_COLUMN_NAME = - // CDC_TYPE_NOT_CDC - val mainDataOutput = targetOutputCols :+ TrueLiteral :+ incrDeletedCountExpr :+ - Literal(CDC_TYPE_NOT_CDC) - if (cdcEnabled) { - // For delete we do a no-op copy with ROW_DELETED_COL = false, INCR_ROW_COUNT_COL as a - // no-op (because the metric will be incremented in `mainDataOutput`) and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_DELETE - val deleteCdcOutput = targetOutputCols :+ FalseLiteral :+ TrueLiteral :+ - Literal(CDC_TYPE_DELETE) - Seq(mainDataOutput, deleteCdcOutput) - } else { - Seq(mainDataOutput) - } - } - exprs.map(resolveOnJoinedPlan) - } - - def notMatchedClauseOutput(clause: DeltaMergeIntoInsertClause): Seq[Seq[Expression]] = { - // Generate insert expressions and set ROW_DELETED_COL = false and - // CDC_TYPE_COLUMN_NAME = CDC_TYPE_NOT_CDC - val insertExprs = clause.resolvedActions.map(_.expr) - val mainDataOutput = resolveOnJoinedPlan( - if (isDeleteWithDuplicateMatchesAndCdc) { - // Must be delete-when-matched merge with duplicate matches + insert clause - // Therefore we must keep the target row id and source row id. Since this is a not-matched - // clause we know the target row-id will be null. See above at - // isDeleteWithDuplicateMatchesAndCdc definition for more details. - insertExprs :+ - Alias(Literal(null), TARGET_ROW_ID_COL)() :+ UnresolvedAttribute(SOURCE_ROW_ID_COL) :+ - FalseLiteral :+ incrInsertedCountExpr :+ Literal(CDC_TYPE_NOT_CDC) - } else { - insertExprs :+ FalseLiteral :+ incrInsertedCountExpr :+ Literal(CDC_TYPE_NOT_CDC) - } - ) - if (cdcEnabled) { - // For insert we have the same expressions as for mainDataOutput, but with - // INCR_ROW_COUNT_COL as a no-op (because the metric will be incremented in - // `mainDataOutput`), and CDC_TYPE_COLUMN_NAME = CDC_TYPE_INSERT - val insertCdcOutput = mainDataOutput.dropRight(2) :+ TrueLiteral :+ Literal(CDC_TYPE_INSERT) - Seq(mainDataOutput, insertCdcOutput) - } else { - Seq(mainDataOutput) - } - } - - def clauseCondition(clause: DeltaMergeIntoClause): Expression = { - // if condition is None, then expression always evaluates to true - val condExpr = clause.condition.getOrElse(TrueLiteral) - resolveOnJoinedPlan(Seq(condExpr)).head - } - - val joinedRowEncoder = RowEncoder(joinedPlan.schema) - val outputRowEncoder = RowEncoder(outputRowSchema).resolveAndBind() - - val processor = new JoinedRowProcessor( - targetRowHasNoMatch = resolveOnJoinedPlan(Seq(col(SOURCE_ROW_PRESENT_COL).isNull.expr)).head, - sourceRowHasNoMatch = resolveOnJoinedPlan(Seq(col(TARGET_ROW_PRESENT_COL).isNull.expr)).head, - matchedConditions = matchedClauses.map(clauseCondition), - matchedOutputs = matchedClauses.map(matchedClauseOutput), - notMatchedConditions = notMatchedClauses.map(clauseCondition), - notMatchedOutputs = notMatchedClauses.map(notMatchedClauseOutput), - noopCopyOutput = resolveOnJoinedPlan( - targetOutputCols :+ FalseLiteral :+ incrNoopCountExpr :+ - Literal(CDC_TYPE_NOT_CDC)), - deleteRowOutput = resolveOnJoinedPlan( - targetOutputCols :+ TrueLiteral :+ TrueLiteral :+ - Literal(CDC_TYPE_NOT_CDC)), - joinedAttributes = joinedPlan.output, - joinedRowEncoder = joinedRowEncoder, - outputRowEncoder = outputRowEncoder - ) - - var outputDF = - Dataset.ofRows(spark, joinedPlan).mapPartitions(processor.processPartition)(outputRowEncoder) - - if (isDeleteWithDuplicateMatchesAndCdc) { - // When we have a delete when matched clause with duplicate matches we have to remove - // duplicate CDC rows. This scenario is further explained at - // isDeleteWithDuplicateMatchesAndCdc definition. - - // To remove duplicate CDC rows generated by the duplicate matches we dedupe by - // TARGET_ROW_ID_COL since there should only be one CDC delete row per target row. - // When there is an insert clause in addition to the delete clause we additionally dedupe by - // SOURCE_ROW_ID_COL and CDC_TYPE_COLUMN_NAME to avoid dropping valid duplicate inserted rows - // and their corresponding CDC rows. - val columnsToDedupeBy = if (notMatchedClauses.nonEmpty) { // insert clause - Seq(TARGET_ROW_ID_COL, SOURCE_ROW_ID_COL, CDC_TYPE_COLUMN_NAME) - } else { - Seq(TARGET_ROW_ID_COL) - } - outputDF = outputDF - .dropDuplicates(columnsToDedupeBy) - .drop(ROW_DROPPED_COL, INCR_ROW_COUNT_COL, TARGET_ROW_ID_COL, SOURCE_ROW_ID_COL) - } else { - outputDF = outputDF.drop(ROW_DROPPED_COL, INCR_ROW_COUNT_COL) - } - - logDebug("writeAllChanges: join output plan:\n" + outputDF.queryExecution) - - // Write to Delta - val newFiles = deltaTxn - .writeFiles(repartitionIfNeeded(spark, outputDF, deltaTxn.metadata.partitionColumns)) - - // Update metrics - val (addedBytes, addedPartitions) = totalBytesAndDistinctPartitionValues(newFiles) - metrics("numTargetFilesAdded") += newFiles.count(_.isInstanceOf[AddFile]) - metrics("numTargetChangeFilesAdded") += newFiles.count(_.isInstanceOf[AddCDCFile]) - metrics("numTargetChangeFileBytes") += newFiles.collect { case f: AddCDCFile => f.size }.sum - metrics("numTargetBytesAdded") += addedBytes - metrics("numTargetPartitionsAddedTo") += addedPartitions - if (multipleMatchDeleteOnlyOvercount.isDefined) { - // Compensate for counting duplicates during the query. - val actualRowsDeleted = - metrics("numTargetRowsDeleted").value - multipleMatchDeleteOnlyOvercount.get - assert(actualRowsDeleted >= 0) - metrics("numTargetRowsDeleted").set(actualRowsDeleted) - } - - newFiles - } - - /** - * Build a new logical plan using the given `files` that has the same output columns (exprIds) as - * the `target` logical plan, so that existing update/insert expressions can be applied on this - * new plan. - */ - private def buildTargetPlanWithFiles( - deltaTxn: OptimisticTransaction, - files: Seq[AddFile]): LogicalPlan = { - val targetOutputCols = getTargetOutputCols(deltaTxn) - val targetOutputColsMap = { - val colsMap: Map[String, NamedExpression] = targetOutputCols.view - .map(col => col.name -> col) - .toMap - if (conf.caseSensitiveAnalysis) { - colsMap - } else { - CaseInsensitiveMap(colsMap) - } - } - - val plan = { - // We have to do surgery to use the attributes from `targetOutputCols` to scan the table. - // In cases of schema evolution, they may not be the same type as the original attributes. - val original = - deltaTxn.deltaLog.createDataFrame(deltaTxn.snapshot, files).queryExecution.analyzed - val transformed = original.transform { - case LogicalRelation(base, output, catalogTbl, isStreaming) => - LogicalRelation( - base, - // We can ignore the new columns which aren't yet AttributeReferences. - targetOutputCols.collect { case a: AttributeReference => a }, - catalogTbl, - isStreaming - ) - } - - // In case of schema evolution & column mapping, we would also need to rebuild the file format - // because under column mapping, the reference schema within DeltaParquetFileFormat - // that is used to populate metadata needs to be updated - if (deltaTxn.metadata.columnMappingMode != NoMapping) { - val updatedFileFormat = deltaTxn.deltaLog.fileFormat(deltaTxn.metadata) - DeltaTableUtils.replaceFileFormat(transformed, updatedFileFormat) - } else { - transformed - } - } - - // For each plan output column, find the corresponding target output column (by name) and - // create an alias - val aliases = plan.output.map { - case newAttrib: AttributeReference => - val existingTargetAttrib = targetOutputColsMap - .get(newAttrib.name) - .getOrElse { - throw DeltaErrors.failedFindAttributeInOutputCollumns( - newAttrib.name, - targetOutputCols.mkString(",")) - } - .asInstanceOf[AttributeReference] - - if (existingTargetAttrib.exprId == newAttrib.exprId) { - // It's not valid to alias an expression to its own exprId (this is considered a - // non-unique exprId by the analyzer), so we just use the attribute directly. - newAttrib - } else { - Alias(newAttrib, existingTargetAttrib.name)(exprId = existingTargetAttrib.exprId) - } - } - - Project(aliases, plan) - } - - /** Expressions to increment SQL metrics */ - private def makeMetricUpdateUDF(name: String): Expression = { - // only capture the needed metric in a local variable - val metric = metrics(name) - udf { () => { metric += 1; true } }.asNondeterministic().apply().expr - } - - private def seqToString(exprs: Seq[Expression]): String = exprs.map(_.sql).mkString("\n\t") - - private def getTargetOutputCols(txn: OptimisticTransaction): Seq[NamedExpression] = { - txn.metadata.schema.map { - col => - targetOutputAttributesMap - .get(col.name) - .map(a => AttributeReference(col.name, col.dataType, col.nullable)(a.exprId)) - .getOrElse(Alias(Literal(null), col.name)()) - } - } - - /** - * Repartitions the output DataFrame by the partition columns if table is partitioned and - * `merge.repartitionBeforeWrite.enabled` is set to true. - */ - protected def repartitionIfNeeded( - spark: SparkSession, - df: DataFrame, - partitionColumns: Seq[String]): DataFrame = { - if (partitionColumns.nonEmpty && spark.conf.get(DeltaSQLConf.MERGE_REPARTITION_BEFORE_WRITE)) { - df.repartition(partitionColumns.map(col): _*) - } else { - df - } - } - - /** - * Execute the given `thunk` and return its result while recording the time taken to do it. - * - * @param sqlMetricName - * name of SQL metric to update with the time taken by the thunk - * @param thunk - * the code to execute - */ - private def recordMergeOperation[A](sqlMetricName: String = null)(thunk: => A): A = { - val startTimeNs = System.nanoTime() - val r = thunk - val timeTakenMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeNs) - if (sqlMetricName != null && timeTakenMs > 0) { - metrics(sqlMetricName) += timeTakenMs - } - r - } -} - -object MergeIntoCommand { - - /** - * Spark UI will track all normal accumulators along with Spark tasks to show them on Web UI. - * However, the accumulator used by `MergeIntoCommand` can store a very large value since it - * tracks all files that need to be rewritten. We should ask Spark UI to not remember it, - * otherwise, the UI data may consume lots of memory. Hence, we use the prefix `internal.metrics.` - * to make this accumulator become an internal accumulator, so that it will not be tracked by - * Spark UI. - */ - val TOUCHED_FILES_ACCUM_NAME = "internal.metrics.MergeIntoDelta.touchedFiles" - - val ROW_ID_COL = "_row_id_" - val TARGET_ROW_ID_COL = "_target_row_id_" - val SOURCE_ROW_ID_COL = "_source_row_id_" - val FILE_NAME_COL = "_file_name_" - val SOURCE_ROW_PRESENT_COL = "_source_row_present_" - val TARGET_ROW_PRESENT_COL = "_target_row_present_" - val ROW_DROPPED_COL = "_row_dropped_" - val INCR_ROW_COUNT_COL = "_incr_row_count_" - - /** - * @param targetRowHasNoMatch - * whether a joined row is a target row with no match in the source table - * @param sourceRowHasNoMatch - * whether a joined row is a source row with no match in the target table - * @param matchedConditions - * condition for each match clause - * @param matchedOutputs - * corresponding output for each match clause. for each clause, we have 1-3 output rows, each of - * which is a sequence of expressions to apply to the joined row - * @param notMatchedConditions - * condition for each not-matched clause - * @param notMatchedOutputs - * corresponding output for each not-matched clause. for each clause, we have 1-2 output rows, - * each of which is a sequence of expressions to apply to the joined row - * @param noopCopyOutput - * no-op expression to copy a target row to the output - * @param deleteRowOutput - * expression to drop a row from the final output. this is used for source rows that don't match - * any not-matched clauses - * @param joinedAttributes - * schema of our outer-joined dataframe - * @param joinedRowEncoder - * joinedDF row encoder - * @param outputRowEncoder - * final output row encoder - */ - class JoinedRowProcessor( - targetRowHasNoMatch: Expression, - sourceRowHasNoMatch: Expression, - matchedConditions: Seq[Expression], - matchedOutputs: Seq[Seq[Seq[Expression]]], - notMatchedConditions: Seq[Expression], - notMatchedOutputs: Seq[Seq[Seq[Expression]]], - noopCopyOutput: Seq[Expression], - deleteRowOutput: Seq[Expression], - joinedAttributes: Seq[Attribute], - joinedRowEncoder: ExpressionEncoder[Row], - outputRowEncoder: ExpressionEncoder[Row]) - extends Serializable { - - private def generateProjection(exprs: Seq[Expression]): UnsafeProjection = { - UnsafeProjection.create(exprs, joinedAttributes) - } - - private def generatePredicate(expr: Expression): BasePredicate = { - GeneratePredicate.generate(expr, joinedAttributes) - } - - def processPartition(rowIterator: Iterator[Row]): Iterator[Row] = { - - val targetRowHasNoMatchPred = generatePredicate(targetRowHasNoMatch) - val sourceRowHasNoMatchPred = generatePredicate(sourceRowHasNoMatch) - val matchedPreds = matchedConditions.map(generatePredicate) - val matchedProjs = matchedOutputs.map(_.map(generateProjection)) - val notMatchedPreds = notMatchedConditions.map(generatePredicate) - val notMatchedProjs = notMatchedOutputs.map(_.map(generateProjection)) - val noopCopyProj = generateProjection(noopCopyOutput) - val deleteRowProj = generateProjection(deleteRowOutput) - val outputProj = UnsafeProjection.create(outputRowEncoder.schema) - - // this is accessing ROW_DROPPED_COL. If ROW_DROPPED_COL is not in outputRowEncoder.schema - // then CDC must be disabled and it's the column after our output cols - def shouldDeleteRow(row: InternalRow): Boolean = { - row.getBoolean( - outputRowEncoder.schema - .getFieldIndex(ROW_DROPPED_COL) - .getOrElse(outputRowEncoder.schema.fields.size) - ) - } - - def processRow(inputRow: InternalRow): Iterator[InternalRow] = { - if (targetRowHasNoMatchPred.eval(inputRow)) { - // Target row did not match any source row, so just copy it to the output - Iterator(noopCopyProj.apply(inputRow)) - } else { - // identify which set of clauses to execute: matched or not-matched ones - val (predicates, projections, noopAction) = if (sourceRowHasNoMatchPred.eval(inputRow)) { - // Source row did not match with any target row, so insert the new source row - (notMatchedPreds, notMatchedProjs, deleteRowProj) - } else { - // Source row matched with target row, so update the target row - (matchedPreds, matchedProjs, noopCopyProj) - } - - // find (predicate, projection) pair whose predicate satisfies inputRow - val pair = - (predicates.zip(projections)).find { case (predicate, _) => predicate.eval(inputRow) } - - pair match { - case Some((_, projections)) => - projections.map(_.apply(inputRow)).iterator - case None => Iterator(noopAction.apply(inputRow)) - } - } - } - - val toRow = joinedRowEncoder.createSerializer() - val fromRow = outputRowEncoder.createDeserializer() - rowIterator - .map(toRow) - .flatMap(processRow) - .filter(!shouldDeleteRow(_)) - .map(notDeletedInternalRow => fromRow(outputProj(notDeletedInternalRow))) - } - } - - /** Count the number of distinct partition values among the AddFiles in the given set. */ - def totalBytesAndDistinctPartitionValues(files: Seq[FileAction]): (Long, Int) = { - val distinctValues = new mutable.HashSet[Map[String, String]]() - var bytes = 0L - val iter = files.collect { case a: AddFile => a }.iterator - while (iter.hasNext) { - val file = iter.next() - distinctValues += file.partitionValues - bytes += file.size - } - // If the only distinct value map is an empty map, then it must be an unpartitioned table. - // Return 0 in that case. - val numDistinctValues = - if (distinctValues.size == 1 && distinctValues.head.isEmpty) 0 else distinctValues.size - (bytes, numDistinctValues) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala deleted file mode 100644 index 6437ab75903..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommand.scala +++ /dev/null @@ -1,484 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.spark.SparkContext -import org.apache.spark.SparkContext.SPARK_JOB_GROUP_ID -import org.apache.spark.sql.{AnalysisException, Encoders, Row, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.{Attribute, AttributeReference, Expression} -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.DeltaOperations.Operation -import org.apache.spark.sql.delta.actions.{Action, AddFile, FileAction, RemoveFile} -import org.apache.spark.sql.delta.commands.OptimizeTableCommandOverwrites.{getDeltaLogClickhouse, groupFilesIntoBinsClickhouse, runOptimizeBinJobClickhouse} -import org.apache.spark.sql.delta.commands.optimize._ -import org.apache.spark.sql.delta.files.SQLMetricsReporting -import org.apache.spark.sql.delta.schema.SchemaUtils -import org.apache.spark.sql.delta.skipping.MultiDimClustering -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.execution.command.{LeafRunnableCommand, RunnableCommand} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.AddMergeTreeParts -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils -import org.apache.spark.sql.execution.metric.SQLMetric -import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric -import org.apache.spark.sql.types._ -import org.apache.spark.util.{SystemClock, ThreadUtils} - -import java.util.ConcurrentModificationException - -import scala.collection.mutable.ArrayBuffer -import scala.collection.parallel.ForkJoinTaskSupport -import scala.collection.parallel.immutable.ParVector - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.0.1. It is modified in: - * 1. getDeltaLogClickhouse 2. runOptimizeBinJobClickhouse 3. groupFilesIntoBinsClickhouse - */ -/** Base class defining abstract optimize command */ -abstract class OptimizeTableCommandBase extends RunnableCommand with DeltaCommand { - - override val output: Seq[Attribute] = Seq( - AttributeReference("path", StringType)(), - AttributeReference("metrics", Encoders.product[OptimizeMetrics].schema)()) - - /** - * Validates ZOrderBy columns - * - validates that partitions columns are not used in `unresolvedZOrderByCols` - * - validates that we already collect stats for all the columns used in - * `unresolvedZOrderByCols` - * - * @param spark - * [[SparkSession]] to use - * @param deltaLog - * [[DeltaLog]] corresponding to the table - * @param unresolvedZOrderByCols - * Seq of [[UnresolvedAttribute]] corresponding to zOrderBy columns - */ - def validateZorderByColumns( - spark: SparkSession, - deltaLog: DeltaLog, - unresolvedZOrderByCols: Seq[UnresolvedAttribute]): Unit = { - if (unresolvedZOrderByCols.isEmpty) return - val metadata = deltaLog.snapshot.metadata - val partitionColumns = metadata.partitionColumns.toSet - val dataSchema = - StructType(metadata.schema.filterNot(c => partitionColumns.contains(c.name))) - val df = spark.createDataFrame(new java.util.ArrayList[Row](), dataSchema) - val checkColStat = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_ZORDER_COL_STAT_CHECK) - val statCollectionSchema = deltaLog.snapshot.statCollectionSchema - val colsWithoutStats = ArrayBuffer[String]() - - unresolvedZOrderByCols.foreach { - colAttribute => - val colName = colAttribute.name - if (checkColStat) { - try { - SchemaUtils.findColumnPosition(colAttribute.nameParts, statCollectionSchema) - } catch { - case e: AnalysisException if e.getMessage.contains("Couldn't find column") => - colsWithoutStats.append(colName) - } - } - val isNameEqual = spark.sessionState.conf.resolver - if (partitionColumns.find(isNameEqual(_, colName)).nonEmpty) { - throw DeltaErrors.zOrderingOnPartitionColumnException(colName) - } - if (df.queryExecution.analyzed.resolve(colAttribute.nameParts, isNameEqual).isEmpty) { - throw DeltaErrors.zOrderingColumnDoesNotExistException(colName) - } - } - if (checkColStat && colsWithoutStats.nonEmpty) { - throw DeltaErrors.zOrderingOnColumnWithNoStatsException(colsWithoutStats.toSeq, spark) - } - } -} - -/** - * The `optimize` command implementation for Spark SQL. Example SQL: - * {{{ - * OPTIMIZE ('/path/to/dir' | delta.table) [WHERE part = 25]; - * }}} - */ -case class OptimizeTableCommand( - path: Option[String], - tableId: Option[TableIdentifier], - partitionPredicate: Option[String])(val zOrderBy: Seq[UnresolvedAttribute]) - extends OptimizeTableCommandBase - with LeafRunnableCommand { - - override val otherCopyArgs: Seq[AnyRef] = zOrderBy :: Nil - - override def run(sparkSession: SparkSession): Seq[Row] = { - // --- modified start - CHDataSourceUtils.ensureClickHouseTableV2(tableId, sparkSession) - val deltaLog = getDeltaLogClickhouse(sparkSession, path, tableId, "OPTIMIZE") - // --- modified end - - val partitionColumns = deltaLog.snapshot.metadata.partitionColumns - // Parse the predicate expression into Catalyst expression and verify only simple filters - // on partition columns are present - val partitionPredicates = partitionPredicate - .map( - predicate => { - val predicates = parsePredicates(sparkSession, predicate) - verifyPartitionPredicates(sparkSession, partitionColumns, predicates) - predicates - }) - .getOrElse(Seq.empty) - - validateZorderByColumns(sparkSession, deltaLog, zOrderBy) - val zOrderByColumns = zOrderBy.map(_.name).toSeq - - new OptimizeExecutor(sparkSession, deltaLog, partitionPredicates, zOrderByColumns) - .optimize() - } -} - -/** - * Optimize job which compacts small files into larger files to reduce the number of files and - * potentially allow more efficient reads. - * - * @param sparkSession - * Spark environment reference. - * @param deltaLog - * Delta table that is being optimized. - * @param partitionPredicate - * List of partition predicates to select subset of files to optimize. - */ -class OptimizeExecutor( - sparkSession: SparkSession, - deltaLog: DeltaLog, - partitionPredicate: Seq[Expression], - zOrderByColumns: Seq[String]) - extends DeltaCommand - with SQLMetricsReporting - with Serializable { - - /** Timestamp to use in [[FileAction]] */ - private val operationTimestamp = new SystemClock().getTimeMillis() - - private val isMultiDimClustering = zOrderByColumns.nonEmpty - - def optimize(): Seq[Row] = { - recordDeltaOperation(deltaLog, "delta.optimize") { - // --- modified start - val isMergeTreeFormat = ClickHouseConfig - .isMergeTreeFormatEngine(deltaLog.snapshot.metadata.configuration) - // --- modified end - val minFileSize = - sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MIN_FILE_SIZE) - val maxFileSize = - sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_FILE_SIZE) - require(minFileSize > 0, "minFileSize must be > 0") - require(maxFileSize > 0, "maxFileSize must be > 0") - - val txn = deltaLog.startTransaction() - if (txn.readVersion == -1) { - throw DeltaErrors.notADeltaTableException(deltaLog.dataPath.toString) - } - - val candidateFiles = txn.filterFiles(partitionPredicate) - val partitionSchema = txn.metadata.partitionSchema - - // select all files in case of multi-dimensional clustering - val filesToProcess = candidateFiles.filter(_.size < minFileSize || isMultiDimClustering) - - // --- modified start - // Create a task pool to parallelize the submission of optimization jobs to Spark. - val threadPool = ThreadUtils.newForkJoinPool( - "OptimizeJob", - sparkSession.sessionState.conf.getConf(DeltaSQLConf.DELTA_OPTIMIZE_MAX_THREADS)) - val (updates, jobs) = if (isMergeTreeFormat) { - val partitionsToCompact = filesToProcess - .groupBy(file => (file.asInstanceOf[AddMergeTreeParts].bucketNum, file.partitionValues)) - .toSeq - - val jobs = groupFilesIntoBinsClickhouse(partitionsToCompact, maxFileSize) - - val parallelJobCollection = new ParVector(jobs.toVector) - - val updates = - try { - val forkJoinPoolTaskSupport = new ForkJoinTaskSupport(threadPool) - parallelJobCollection.tasksupport = forkJoinPoolTaskSupport - - parallelJobCollection - .flatMap( - partitionBinGroup => - runOptimizeBinJobClickhouse( - txn, - partitionBinGroup._1._2, - partitionBinGroup._1._1, - partitionBinGroup._2, - maxFileSize)) - .seq - } finally { - threadPool.shutdownNow() - } - (updates, jobs) - } else { - val partitionsToCompact = filesToProcess.groupBy(_.partitionValues).toSeq - - val jobs = groupFilesIntoBins(partitionsToCompact, maxFileSize) - - val parallelJobCollection = new ParVector(jobs.toVector) - - val updates = try { - val forkJoinPoolTaskSupport = new ForkJoinTaskSupport(threadPool) - parallelJobCollection.tasksupport = forkJoinPoolTaskSupport - - parallelJobCollection.flatMap(partitionBinGroup => - runOptimizeBinJob(txn, partitionBinGroup._1, partitionBinGroup._2, maxFileSize)).seq - } finally { - threadPool.shutdownNow() - } - (updates, jobs) - } - // --- modified end - - val addedFiles = updates.collect { case a: AddFile => a } - val removedFiles = updates.collect { case r: RemoveFile => r } - if (addedFiles.size > 0) { - val operation = DeltaOperations.Optimize(partitionPredicate.map(_.sql), zOrderByColumns) - val metrics = createMetrics(sparkSession.sparkContext, addedFiles, removedFiles) - commitAndRetry(txn, operation, updates, metrics) { - newTxn => - val newPartitionSchema = newTxn.metadata.partitionSchema - val candidateSetOld = candidateFiles.map(_.path).toSet - val candidateSetNew = newTxn.filterFiles(partitionPredicate).map(_.path).toSet - - // As long as all of the files that we compacted are still part of the table, - // and the partitioning has not changed it is valid to continue to try - // and commit this checkpoint. - if ( - candidateSetOld.subsetOf(candidateSetNew) && partitionSchema == newPartitionSchema - ) { - true - } else { - val deleted = candidateSetOld -- candidateSetNew - logWarning( - s"The following compacted files were delete " + - s"during checkpoint ${deleted.mkString(",")}. Aborting the compaction.") - false - } - } - } - - val optimizeStats = OptimizeStats() - optimizeStats.addedFilesSizeStats.merge(addedFiles) - optimizeStats.removedFilesSizeStats.merge(removedFiles) - optimizeStats.numPartitionsOptimized = jobs.map(j => j._1).distinct.size - optimizeStats.numBatches = jobs.size - optimizeStats.totalConsideredFiles = candidateFiles.size - optimizeStats.totalFilesSkipped = optimizeStats.totalConsideredFiles - removedFiles.size - - if (isMultiDimClustering) { - val inputFileStats = - ZOrderFileStats(removedFiles.size, removedFiles.map(_.size.getOrElse(0L)).sum) - optimizeStats.zOrderStats = Some( - ZOrderStats( - strategyName = "all", // means process all files in a partition - inputCubeFiles = ZOrderFileStats(0, 0), - inputOtherFiles = inputFileStats, - inputNumCubes = 0, - mergedFiles = inputFileStats, - // There will one z-cube for each partition - numOutputCubes = optimizeStats.numPartitionsOptimized - )) - } - - return Seq(Row(deltaLog.dataPath.toString, optimizeStats.toOptimizeMetrics)) - } - } - - /** - * Utility methods to group files into bins for optimize. - * - * @param partitionsToCompact - * List of files to compact group by partition. Partition is defined by the partition values - * (partCol -> partValue) - * @param maxTargetFileSize - * Max size (in bytes) of the compaction output file. - * @return - * Sequence of bins. Each bin contains one or more files from the same partition and targeted - * for one output file. - */ - private def groupFilesIntoBins( - partitionsToCompact: Seq[(Map[String, String], Seq[AddFile])], - maxTargetFileSize: Long): Seq[(Map[String, String], Seq[AddFile])] = { - partitionsToCompact.flatMap { - case (partition, files) => - val bins = new ArrayBuffer[Seq[AddFile]]() - - val currentBin = new ArrayBuffer[AddFile]() - var currentBinSize = 0L - - files.sortBy(_.size).foreach { - file => - // Generally, a bin is a group of existing files, whose total size does not exceed the - // desired maxFileSize. They will be coalesced into a single output file. - // However, if isMultiDimClustering = true, all files in a partition will be read by the - // same job, the data will be range-partitioned and - // numFiles = totalFileSize / maxFileSize will be produced. See below. - if (file.size + currentBinSize > maxTargetFileSize && !isMultiDimClustering) { - bins += currentBin.toVector - currentBin.clear() - currentBin += file - currentBinSize = file.size - } else { - currentBin += file - currentBinSize += file.size - } - } - - if (currentBin.nonEmpty) { - bins += currentBin.toVector - } - - bins - .map(b => (partition, b)) - // select bins that have at least two files or in case of multi-dim clustering - // select all bins - .filter(_._2.size > 1 || isMultiDimClustering) - } - } - - /** - * Utility method to run a Spark job to compact the files in given bin - * - * @param txn - * [[OptimisticTransaction]] instance in use to commit the changes to DeltaLog. - * @param partition - * Partition values of the partition that files in [[bin]] belongs to. - * @param bin - * List of files to compact into one large file. - * @param maxFileSize - * Targeted output file size in bytes - */ - private def runOptimizeBinJob( - txn: OptimisticTransaction, - partition: Map[String, String], - bin: Seq[AddFile], - maxFileSize: Long): Seq[FileAction] = { - val baseTablePath = deltaLog.dataPath - - val input = txn.deltaLog.createDataFrame(txn.snapshot, bin, actionTypeOpt = Some("Optimize")) - val repartitionDF = if (isMultiDimClustering) { - val totalSize = bin.map(_.size).sum - val approxNumFiles = Math.max(1, totalSize / maxFileSize).toInt - MultiDimClustering.cluster(input, approxNumFiles, zOrderByColumns) - } else { - input.coalesce(numPartitions = 1) - } - - val partitionDesc = partition.toSeq.map(entry => entry._1 + "=" + entry._2).mkString(",") - - val partitionName = if (partition.isEmpty) "" else s" in partition ($partitionDesc)" - val description = s"$baseTablePath
Optimizing ${bin.size} files" + partitionName - sparkSession.sparkContext.setJobGroup( - sparkSession.sparkContext.getLocalProperty(SPARK_JOB_GROUP_ID), - description) - - val addFiles = txn.writeFiles(repartitionDF).collect { - case a: AddFile => - a.copy(dataChange = false) - case other => - throw new IllegalStateException( - s"Unexpected action $other with type ${other.getClass}. File compaction job output" + - s"should only have AddFiles") - } - val removeFiles = bin.map(f => f.removeWithTimestamp(operationTimestamp, dataChange = false)) - val updates = addFiles ++ removeFiles - updates - } - - /** - * Attempts to commit the given actions to the log. In the case of a concurrent update, the given - * function will be invoked with a new transaction to allow custom conflict detection logic to - * indicate it is safe to try again, by returning `true`. - * - * This function will continue to try to commit to the log as long as `f` returns `true`, - * otherwise throws a subclass of [[ConcurrentModificationException]]. - */ - private def commitAndRetry( - txn: OptimisticTransaction, - optimizeOperation: Operation, - actions: Seq[Action], - metrics: Map[String, SQLMetric])(f: OptimisticTransaction => Boolean): Unit = { - try { - txn.registerSQLMetrics(sparkSession, metrics) - txn.commit(actions, optimizeOperation) - } catch { - case e: ConcurrentModificationException => - val newTxn = txn.deltaLog.startTransaction() - if (f(newTxn)) { - logInfo("Retrying commit after checking for semantic conflicts with concurrent updates.") - commitAndRetry(newTxn, optimizeOperation, actions, metrics)(f) - } else { - logWarning("Semantic conflicts detected. Aborting operation.") - throw e - } - } - } - - /** Create a map of SQL metrics for adding to the commit history. */ - private def createMetrics( - sparkContext: SparkContext, - addedFiles: Seq[AddFile], - removedFiles: Seq[RemoveFile]): Map[String, SQLMetric] = { - - def setAndReturnMetric(description: String, value: Long) = { - val metric = createMetric(sparkContext, description) - metric.set(value) - metric - } - - def totalSize(actions: Seq[FileAction]): Long = { - var totalSize = 0L - actions.foreach { - file => - val fileSize = file match { - case addFile: AddFile => addFile.size - case removeFile: RemoveFile => removeFile.size.getOrElse(0L) - case default => - throw new IllegalArgumentException(s"Unknown FileAction type: ${default.getClass}") - } - totalSize += fileSize - } - totalSize - } - - val sizeStats = FileSizeStatsWithHistogram.create(addedFiles.map(_.size).sorted) - Map[String, SQLMetric]( - "minFileSize" -> setAndReturnMetric("minimum file size", sizeStats.get.min), - "p25FileSize" -> setAndReturnMetric("25th percentile file size", sizeStats.get.p25), - "p50FileSize" -> setAndReturnMetric("50th percentile file size", sizeStats.get.p50), - "p75FileSize" -> setAndReturnMetric("75th percentile file size", sizeStats.get.p75), - "maxFileSize" -> setAndReturnMetric("maximum file size", sizeStats.get.max), - "numAddedFiles" -> setAndReturnMetric("total number of files added.", addedFiles.size), - "numRemovedFiles" -> setAndReturnMetric("total number of files removed.", removedFiles.size), - "numAddedBytes" -> setAndReturnMetric("total number of bytes added", totalSize(addedFiles)), - "numRemovedBytes" -> - setAndReturnMetric("total number of bytes removed", totalSize(removedFiles)) - ) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala deleted file mode 100644 index 6170a0c3101..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/OptimizeTableCommandOverwrites.scala +++ /dev/null @@ -1,323 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.gluten.memory.CHThreadGroup -import org.apache.spark.{TaskContext, TaskOutputFileAlreadyExistException} -import org.apache.spark.internal.Logging -import org.apache.spark.internal.io.FileCommitProtocol.TaskCommitMessage -import org.apache.spark.internal.io.SparkHadoopWriterUtils -import org.apache.spark.shuffle.FetchFailedException -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} -import org.apache.spark.sql.catalyst.catalog.CatalogTableType -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{AddFile, FileAction} -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.errors.QueryExecutionErrors -import org.apache.spark.sql.execution.datasources.{CHDatasourceJniWrapper, WriteTaskResult} -import org.apache.spark.sql.execution.datasources.v1.CHMergeTreeWriterInjects -import org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata.{AddFileTags, AddMergeTreeParts} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.StructType -import org.apache.spark.util.{SerializableConfiguration, SystemClock, Utils} -import org.apache.hadoop.fs.{FileAlreadyExistsException, Path} -import org.apache.hadoop.mapreduce.{TaskAttemptContext, TaskAttemptID, TaskID, TaskType} -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl - -import java.util.Date -import scala.collection.mutable.ArrayBuffer - -object OptimizeTableCommandOverwrites extends Logging { - - case class TaskDescription( - path: String, - database: String, - tableName: String, - snapshotId: String, - orderByKey: String, - lowCardKey: String, - minmaxIndexKey: String, - bfIndexKey: String, - setIndexKey: String, - primaryKey: String, - partitionColumns: Seq[String], - partList: Seq[String], - tableSchema: StructType, - clickhouseTableConfigs: Map[String, String], - serializableHadoopConf: SerializableConfiguration, - jobIdInstant: Long, - partitionDir: Option[String], - bucketDir: Option[String] - ) - - private def executeTask( - description: TaskDescription, - sparkStageId: Int, - sparkPartitionId: Int, - sparkAttemptNumber: Int - ): WriteTaskResult = { - CHThreadGroup.registerNewThreadGroup() - val jobId = SparkHadoopWriterUtils.createJobID(new Date(description.jobIdInstant), sparkStageId) - val taskId = new TaskID(jobId, TaskType.MAP, sparkPartitionId) - val taskAttemptId = new TaskAttemptID(taskId, sparkAttemptNumber) - - // Set up the attempt context required to use in the output committer. - val taskAttemptContext: TaskAttemptContext = { - // Set up the configuration object - val hadoopConf = description.serializableHadoopConf.value - hadoopConf.set("mapreduce.job.id", jobId.toString) - hadoopConf.set("mapreduce.task.id", taskAttemptId.getTaskID.toString) - hadoopConf.set("mapreduce.task.attempt.id", taskAttemptId.toString) - hadoopConf.setBoolean("mapreduce.task.ismap", true) - hadoopConf.setInt("mapreduce.task.partition", 0) - - new TaskAttemptContextImpl(hadoopConf, taskAttemptId) - } - - try { - Utils.tryWithSafeFinallyAndFailureCallbacks(block = { - - val planWithSplitInfo = CHMergeTreeWriterInjects.genMergeTreeWriteRel( - description.path, - description.database, - description.tableName, - description.snapshotId, - description.orderByKey, - description.lowCardKey, - description.minmaxIndexKey, - description.bfIndexKey, - description.setIndexKey, - description.primaryKey, - description.partitionColumns, - description.partList, - description.tableSchema, - description.clickhouseTableConfigs, - description.tableSchema.toAttributes - ) - - val returnedMetrics = - CHDatasourceJniWrapper.nativeMergeMTParts( - planWithSplitInfo.splitInfo, - description.partitionDir.getOrElse(""), - description.bucketDir.getOrElse("") - ) - if (returnedMetrics != null && returnedMetrics.nonEmpty) { - val addFiles = AddFileTags.partsMetricsToAddFile( - description.database, - description.tableName, - description.path, - returnedMetrics, - Seq(Utils.localHostName())) - - val (taskCommitMessage, taskCommitTime) = Utils.timeTakenMs { - // committer.commitTask(taskAttemptContext) - new TaskCommitMessage(addFiles.toSeq) - } - -// val summary = MergeTreeExecutedWriteSummary( -// updatedPartitions = updatedPartitions.toSet, -// stats = statsTrackers.map(_.getFinalStats(taskCommitTime))) - WriteTaskResult(taskCommitMessage, null) - } else { - throw new IllegalStateException() - } - })( - catchBlock = { - // If there is an error, abort the task - logError(s"Job $jobId aborted.") - }, - finallyBlock = {}) - } catch { - case e: FetchFailedException => - throw e - case f: FileAlreadyExistsException if SQLConf.get.fastFailFileFormatOutput => - // If any output file to write already exists, it does not make sense to re-run this task. - // We throw the exception and let Executor throw ExceptionFailure to abort the job. - throw new TaskOutputFileAlreadyExistException(f) - case t: Throwable => - throw QueryExecutionErrors.taskFailedWhileWritingRowsError(t) - } - - } - - def runOptimizeBinJobClickhouse( - txn: OptimisticTransaction, - partitionValues: Map[String, String], - bucketNum: String, - bin: Seq[AddFile], - maxFileSize: Long): Seq[FileAction] = { - val tableV2 = ClickHouseTableV2.getTable(txn.deltaLog); - - val sparkSession = SparkSession.getActiveSession.get - - val rddWithNonEmptyPartitions = - sparkSession.sparkContext.parallelize(Array.empty[InternalRow], 1) - - val jobIdInstant = new Date().getTime - val ret = new Array[WriteTaskResult](rddWithNonEmptyPartitions.partitions.length) - - val serializableHadoopConf = new SerializableConfiguration( - sparkSession.sessionState.newHadoopConfWithOptions( - txn.metadata.configuration ++ txn.deltaLog.options)) - - val partitionDir = if (tableV2.partitionColumns.isEmpty) { - None - } else { - Some(tableV2.partitionColumns.map(c => c + "=" + partitionValues(c)).mkString("/")) - } - - val bucketDir = if (tableV2.bucketOption.isEmpty) { - None - } else { - Some(bucketNum) - } - - val description = TaskDescription.apply( - txn.deltaLog.dataPath.toString, - tableV2.dataBaseName, - tableV2.tableName, - ClickhouseSnapshot.genSnapshotId(tableV2.snapshot), - tableV2.orderByKey, - tableV2.lowCardKey, - tableV2.minmaxIndexKey, - tableV2.bfIndexKey, - tableV2.setIndexKey, - tableV2.primaryKey, - tableV2.partitionColumns, - bin.map(_.asInstanceOf[AddMergeTreeParts].name), - tableV2.schema(), - tableV2.clickhouseTableConfigs, - serializableHadoopConf, - jobIdInstant, - partitionDir, - bucketDir - ) - sparkSession.sparkContext.runJob( - rddWithNonEmptyPartitions, - (taskContext: TaskContext, _: Iterator[InternalRow]) => { - executeTask( - description, - taskContext.stageId(), - taskContext.partitionId(), - taskContext.taskAttemptId().toInt & Integer.MAX_VALUE - ) - }, - rddWithNonEmptyPartitions.partitions.indices, - (index, res: WriteTaskResult) => { - ret(index) = res - } - ) - - val addFiles = ret - .flatMap(_.commitMsg.obj.asInstanceOf[Seq[AddFile]]) - .toSeq - - val removeFiles = - bin.map(f => f.removeWithTimestamp(new SystemClock().getTimeMillis(), dataChange = false)) - addFiles ++ removeFiles - - } - - def getDeltaLogClickhouse( - spark: SparkSession, - path: Option[String], - tableIdentifier: Option[TableIdentifier], - operationName: String, - hadoopConf: Map[String, String] = Map.empty): DeltaLog = { - val tablePath = - if (path.nonEmpty) { - new Path(path.get) - } else if (tableIdentifier.nonEmpty) { - val sessionCatalog = spark.sessionState.catalog - lazy val metadata = sessionCatalog.getTableMetadata(tableIdentifier.get) - - if (CHDataSourceUtils.isClickhousePath(spark, tableIdentifier.get)) { - new Path(tableIdentifier.get.table) - } else if (CHDataSourceUtils.isClickHouseTable(spark, tableIdentifier.get)) { - new Path(metadata.location) - } else { - DeltaTableIdentifier(spark, tableIdentifier.get) match { - case Some(id) if id.path.nonEmpty => - new Path(id.path.get) - case Some(id) if id.table.nonEmpty => - new Path(metadata.location) - case _ => - if (metadata.tableType == CatalogTableType.VIEW) { - throw DeltaErrors.viewNotSupported(operationName) - } - throw DeltaErrors.notADeltaTableException(operationName) - } - } - } else { - throw DeltaErrors.missingTableIdentifierException(operationName) - } - - val startTime = Some(System.currentTimeMillis) - val deltaLog = DeltaLog.forTable(spark, tablePath, hadoopConf) - if (deltaLog.update(checkIfUpdatedSinceTs = startTime).version < 0) { - throw DeltaErrors.notADeltaTableException( - operationName, - DeltaTableIdentifier(path, tableIdentifier)) - } - deltaLog - } - - def groupFilesIntoBinsClickhouse( - partitionsToCompact: Seq[((String, Map[String, String]), Seq[AddFile])], - maxTargetFileSize: Long): Seq[((String, Map[String, String]), Seq[AddFile])] = { - partitionsToCompact.flatMap { - case (partition, files) => - val bins = new ArrayBuffer[Seq[AddFile]]() - - val currentBin = new ArrayBuffer[AddFile]() - var currentBinSize = 0L - - files.sortBy(_.size).foreach { - file => - // Generally, a bin is a group of existing files, whose total size does not exceed the - // desired maxFileSize. They will be coalesced into a single output file. - // However, if isMultiDimClustering = true, all files in a partition will be read by the - // same job, the data will be range-partitioned and - // numFiles = totalFileSize / maxFileSize - // will be produced. See below. - - // isMultiDimClustering is always false for Gluten Clickhouse for now - if (file.size + currentBinSize > maxTargetFileSize /* && !isMultiDimClustering */ ) { - bins += currentBin.toVector - currentBin.clear() - currentBin += file - currentBinSize = file.size - } else { - currentBin += file - currentBinSize += file.size - } - } - - if (currentBin.nonEmpty) { - bins += currentBin.toVector - } - - bins - .map(b => (partition, b)) - // select bins that have at least two files or in case of multi-dim clustering - // select all bins - .filter(_._2.size > 1 /* || isMultiDimClustering */ ) - } - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala deleted file mode 100644 index ce6a7a474ec..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/UpdateCommand.scala +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.spark.SparkContext -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute -import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, If, Literal} -import org.apache.spark.sql.catalyst.plans.QueryPlan -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{AddCDCFile, AddFile, FileAction} -import org.apache.spark.sql.delta.commands.cdc.CDCReader.{CDC_TYPE_COLUMN_NAME, CDC_TYPE_NOT_CDC, CDC_TYPE_UPDATE_POSTIMAGE, CDC_TYPE_UPDATE_PREIMAGE} -import org.apache.spark.sql.delta.files.{TahoeBatchFileIndex, TahoeFileIndex} -import org.apache.spark.sql.execution.SQLExecution -import org.apache.spark.sql.execution.command.LeafRunnableCommand -import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} -import org.apache.spark.sql.execution.metric.SQLMetrics.createMetric -import org.apache.spark.sql.functions._ - -// scalastyle:off import.ordering.noEmptyLine -import org.apache.hadoop.fs.Path - -/** - * Gluten overwrite Delta: - * - * This file is copied from Delta 2.2.0. - */ - -/** - * Performs an Update using `updateExpression` on the rows that match `condition` - * - * Algorithm: 1) Identify the affected files, i.e., the files that may have the rows to be updated. - * 2) Scan affected files, apply the updates, and generate a new DF with updated rows. 3) Use the - * Delta protocol to atomically write the new DF as new files and remove the affected files that are - * identified in step 1. - */ -case class UpdateCommand( - tahoeFileIndex: TahoeFileIndex, - target: LogicalPlan, - updateExpressions: Seq[Expression], - condition: Option[Expression]) - extends LeafRunnableCommand - with DeltaCommand { - - override def innerChildren: Seq[QueryPlan[_]] = Seq(target) - - @transient private lazy val sc: SparkContext = SparkContext.getOrCreate() - - override lazy val metrics = Map[String, SQLMetric]( - "numAddedFiles" -> createMetric(sc, "number of files added."), - "numRemovedFiles" -> createMetric(sc, "number of files removed."), - "numUpdatedRows" -> createMetric(sc, "number of rows updated."), - "numCopiedRows" -> createMetric(sc, "number of rows copied."), - "executionTimeMs" -> createMetric(sc, "time taken to execute the entire operation"), - "scanTimeMs" -> createMetric(sc, "time taken to scan the files for matches"), - "rewriteTimeMs" -> createMetric(sc, "time taken to rewrite the matched files"), - "numAddedChangeFiles" -> createMetric(sc, "number of change data capture files generated"), - "changeFileBytes" -> createMetric(sc, "total size of change data capture files generated"), - "numTouchedRows" -> createMetric(sc, "number of rows touched (copied + updated)") - ) - - final override def run(sparkSession: SparkSession): Seq[Row] = { - recordDeltaOperation(tahoeFileIndex.deltaLog, "delta.dml.update") { - val deltaLog = tahoeFileIndex.deltaLog - deltaLog.assertRemovable() - deltaLog.withNewTransaction(txn => performUpdate(sparkSession, deltaLog, txn)) - // Re-cache all cached plans(including this relation itself, if it's cached) that refer to - // this data source relation. - sparkSession.sharedState.cacheManager.recacheByPlan(sparkSession, target) - } - Seq.empty[Row] - } - - private def performUpdate( - sparkSession: SparkSession, - deltaLog: DeltaLog, - txn: OptimisticTransaction): Unit = { - import sparkSession.implicits._ - - var numTouchedFiles: Long = 0 - var numRewrittenFiles: Long = 0 - var numAddedChangeFiles: Long = 0 - var changeFileBytes: Long = 0 - var scanTimeMs: Long = 0 - var rewriteTimeMs: Long = 0 - - val startTime = System.nanoTime() - val numFilesTotal = deltaLog.snapshot.numOfFiles - - val updateCondition = condition.getOrElse(Literal.TrueLiteral) - val (metadataPredicates, dataPredicates) = - DeltaTableUtils.splitMetadataAndDataPredicates( - updateCondition, - txn.metadata.partitionColumns, - sparkSession) - val candidateFiles = txn.filterFiles(metadataPredicates ++ dataPredicates) - val nameToAddFile = generateCandidateFileMap(deltaLog.dataPath, candidateFiles) - - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - - val filesToRewrite: Seq[AddFile] = if (candidateFiles.isEmpty) { - // Case 1: Do nothing if no row qualifies the partition predicates - // that are part of Update condition - Nil - } else if (dataPredicates.isEmpty) { - // Case 2: Update all the rows from the files that are in the specified partitions - // when the data filter is empty - candidateFiles - } else { - // Case 3: Find all the affected files using the user-specified condition - val fileIndex = new TahoeBatchFileIndex( - sparkSession, - "update", - candidateFiles, - deltaLog, - tahoeFileIndex.path, - txn.snapshot) - // Keep everything from the resolved target except a new TahoeFileIndex - // that only involves the affected files instead of all files. - val newTarget = DeltaTableUtils.replaceFileIndex(target, fileIndex) - val data = Dataset.ofRows(sparkSession, newTarget) - val updatedRowCount = metrics("numUpdatedRows") - val updatedRowUdf = udf { - () => - updatedRowCount += 1 - true - }.asNondeterministic() - val pathsToRewrite = - withStatusCode("DELTA", UpdateCommand.FINDING_TOUCHED_FILES_MSG) { - // --- modified start - data - .filter(new Column(updateCondition)) - .filter(updatedRowUdf()) - .select(input_file_name().as("input_files")) - .distinct() - .as[String] - .collect() - // --- modified end - } - - scanTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - - pathsToRewrite.map(getTouchedFile(deltaLog.dataPath, _, nameToAddFile)).toSeq - } - - numTouchedFiles = filesToRewrite.length - - val newActions = if (filesToRewrite.isEmpty) { - // Do nothing if no row qualifies the UPDATE condition - Nil - } else { - // Generate the new files containing the updated values - withStatusCode("DELTA", UpdateCommand.rewritingFilesMsg(filesToRewrite.size)) { - rewriteFiles( - sparkSession, - txn, - tahoeFileIndex.path, - filesToRewrite.map(_.path), - nameToAddFile, - updateCondition) - } - } - - rewriteTimeMs = (System.nanoTime() - startTime) / 1000 / 1000 - scanTimeMs - - val (changeActions, addActions) = newActions.partition(_.isInstanceOf[AddCDCFile]) - numRewrittenFiles = addActions.size - numAddedChangeFiles = changeActions.size - changeFileBytes = changeActions.collect { case f: AddCDCFile => f.size }.sum - - val totalActions = if (filesToRewrite.isEmpty) { - // Do nothing if no row qualifies the UPDATE condition - Nil - } else { - // Delete the old files and return those delete actions along with the new AddFile actions for - // files containing the updated values - val operationTimestamp = System.currentTimeMillis() - val deleteActions = filesToRewrite.map(_.removeWithTimestamp(operationTimestamp)) - - deleteActions ++ newActions - } - - if (totalActions.nonEmpty) { - metrics("numAddedFiles").set(numRewrittenFiles) - metrics("numAddedChangeFiles").set(numAddedChangeFiles) - metrics("changeFileBytes").set(changeFileBytes) - metrics("numRemovedFiles").set(numTouchedFiles) - metrics("executionTimeMs").set((System.nanoTime() - startTime) / 1000 / 1000) - metrics("scanTimeMs").set(scanTimeMs) - metrics("rewriteTimeMs").set(rewriteTimeMs) - // In the case where the numUpdatedRows is not captured, we can siphon out the metrics from - // the BasicWriteStatsTracker. This is for case 2 where the update condition contains only - // metadata predicates and so the entire partition is re-written. - val outputRows = txn.getMetric("numOutputRows").map(_.value).getOrElse(-1L) - if ( - metrics("numUpdatedRows").value == 0 && outputRows != 0 && - metrics("numCopiedRows").value == 0 - ) { - // We know that numTouchedRows = numCopiedRows + numUpdatedRows. - // Since an entire partition was re-written, no rows were copied. - // So numTouchedRows == numUpdateRows - metrics("numUpdatedRows").set(metrics("numTouchedRows").value) - } else { - // This is for case 3 where the update condition contains both metadata and data predicates - // so relevant files will have some rows updated and some rows copied. We don't need to - // consider case 1 here, where no files match the update condition, as we know that - // `totalActions` is empty. - metrics("numCopiedRows").set( - metrics("numTouchedRows").value - metrics("numUpdatedRows").value) - } - txn.registerSQLMetrics(sparkSession, metrics) - txn.commit(totalActions, DeltaOperations.Update(condition.map(_.toString))) - // This is needed to make the SQL metrics visible in the Spark UI - val executionId = sparkSession.sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) - SQLMetrics.postDriverMetricUpdates( - sparkSession.sparkContext, - executionId, - metrics.values.toSeq) - } - - recordDeltaEvent( - deltaLog, - "delta.dml.update.stats", - data = UpdateMetric( - condition = condition.map(_.sql).getOrElse("true"), - numFilesTotal, - numTouchedFiles, - numRewrittenFiles, - numAddedChangeFiles, - changeFileBytes, - scanTimeMs, - rewriteTimeMs - ) - ) - } - - /** - * Scan all the affected files and write out the updated files. - * - * When CDF is enabled, includes the generation of CDC preimage and postimage columns for changed - * rows. - * - * @return - * the list of [[AddFile]]s and [[AddCDCFile]]s that have been written. - */ - private def rewriteFiles( - spark: SparkSession, - txn: OptimisticTransaction, - rootPath: Path, - inputLeafFiles: Seq[String], - nameToAddFileMap: Map[String, AddFile], - condition: Expression): Seq[FileAction] = { - // Containing the map from the relative file path to AddFile - val baseRelation = - buildBaseRelation(spark, txn, "update", rootPath, inputLeafFiles, nameToAddFileMap) - val newTarget = DeltaTableUtils.replaceFileIndex(target, baseRelation.location) - val targetDf = Dataset.ofRows(spark, newTarget) - - // Number of total rows that we have seen, i.e. are either copying or updating (sum of both). - // This will be used later, along with numUpdatedRows, to determine numCopiedRows. - val numTouchedRows = metrics("numTouchedRows") - val numTouchedRowsUdf = udf { - () => - numTouchedRows += 1 - true - }.asNondeterministic() - - val updatedDataFrame = UpdateCommand.withUpdatedColumns( - target, - updateExpressions, - condition, - targetDf - .filter(numTouchedRowsUdf()) - .withColumn(UpdateCommand.CONDITION_COLUMN_NAME, new Column(condition)), - UpdateCommand.shouldOutputCdc(txn) - ) - - txn.writeFiles(updatedDataFrame) - } -} - -object UpdateCommand { - val FILE_NAME_COLUMN = "_input_file_name_" - val CONDITION_COLUMN_NAME = "__condition__" - val FINDING_TOUCHED_FILES_MSG: String = "Finding files to rewrite for UPDATE operation" - - def rewritingFilesMsg(numFilesToRewrite: Long): String = - s"Rewriting $numFilesToRewrite files for UPDATE operation" - - /** - * Whether or not CDC is enabled on this table and, thus, if we should output CDC data during this - * UPDATE operation. - */ - def shouldOutputCdc(txn: OptimisticTransaction): Boolean = { - DeltaConfigs.CHANGE_DATA_FEED.fromMetaData(txn.metadata) - } - - /** - * Build the new columns. If the condition matches, generate the new value using the corresponding - * UPDATE EXPRESSION; otherwise, keep the original column value. - * - * When CDC is enabled, includes the generation of CDC pre-image and post-image columns for - * changed rows. - * - * @param target - * target we are updating into - * @param updateExpressions - * the update transformation to perform on the input DataFrame - * @param dfWithEvaluatedCondition - * source DataFrame on which we will apply the update expressions with an additional column - * CONDITION_COLUMN_NAME which is the true/false value of if the update condition is satisfied - * @param condition - * update condition - * @param shouldOutputCdc - * if we should output CDC data during this UPDATE operation. - * @return - * the updated DataFrame, with extra CDC columns if CDC is enabled - */ - def withUpdatedColumns( - target: LogicalPlan, - updateExpressions: Seq[Expression], - condition: Expression, - dfWithEvaluatedCondition: DataFrame, - shouldOutputCdc: Boolean): DataFrame = { - val resultDf = if (shouldOutputCdc) { - val namedUpdateCols = updateExpressions.zip(target.output).map { - case (expr, targetCol) => new Column(expr).as(targetCol.name) - } - - // Build an array of output rows to be unpacked later. If the condition is matched, we - // generate CDC pre and postimages in addition to the final output row; if the condition - // isn't matched, we just generate a rewritten no-op row without any CDC events. - val preimageCols = target.output.map(new Column(_)) :+ - lit(CDC_TYPE_UPDATE_PREIMAGE).as(CDC_TYPE_COLUMN_NAME) - val postimageCols = namedUpdateCols :+ - lit(CDC_TYPE_UPDATE_POSTIMAGE).as(CDC_TYPE_COLUMN_NAME) - val updatedDataCols = namedUpdateCols :+ - typedLit[String](CDC_TYPE_NOT_CDC).as(CDC_TYPE_COLUMN_NAME) - val noopRewriteCols = target.output.map(new Column(_)) :+ - typedLit[String](CDC_TYPE_NOT_CDC).as(CDC_TYPE_COLUMN_NAME) - val packedUpdates = array( - struct(preimageCols: _*), - struct(postimageCols: _*), - struct(updatedDataCols: _*) - ).expr - - val packedData = if (condition == Literal.TrueLiteral) { - packedUpdates - } else { - If( - UnresolvedAttribute(CONDITION_COLUMN_NAME), - packedUpdates, // if it should be updated, then use `packagedUpdates` - array(struct(noopRewriteCols: _*)).expr - ) // else, this is a noop rewrite - } - - // Explode the packed array, and project back out the final data columns. - val finalColNames = target.output.map(_.name) :+ CDC_TYPE_COLUMN_NAME - dfWithEvaluatedCondition - .select(explode(new Column(packedData)).as("packedData")) - .select(finalColNames.map(n => col(s"packedData.`$n`").as(s"$n")): _*) - } else { - val finalCols = updateExpressions.zip(target.output).map { - case (update, original) => - val updated = if (condition == Literal.TrueLiteral) { - update - } else { - If(UnresolvedAttribute(CONDITION_COLUMN_NAME), update, original) - } - new Column(Alias(updated, original.name)()) - } - - dfWithEvaluatedCondition.select(finalCols: _*) - } - - resultDf.drop(CONDITION_COLUMN_NAME) - } -} - -/** - * Used to report details about update. - * - * @param condition: - * what was the update condition - * @param numFilesTotal: - * how big is the table - * @param numTouchedFiles: - * how many files did we touch - * @param numRewrittenFiles: - * how many files had to be rewritten - * @param numAddedChangeFiles: - * how many change files were generated - * @param changeFileBytes: - * total size of change files generated - * @param scanTimeMs: - * how long did finding take - * @param rewriteTimeMs: - * how long did rewriting take - * - * @note - * All the time units are milliseconds. - */ -case class UpdateMetric( - condition: String, - numFilesTotal: Long, - numTouchedFiles: Long, - numRewrittenFiles: Long, - numAddedChangeFiles: Long, - changeFileBytes: Long, - scanTimeMs: Long, - rewriteTimeMs: Long -) diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala deleted file mode 100644 index 939e6bcbf29..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/commands/VacuumCommand.scala +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.commands - -import org.apache.gluten.extension.GlutenSessionExtensions - -import org.apache.spark.sql.{DataFrame, SparkSession} -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.actions.{FileAction, RemoveFile} -import org.apache.spark.sql.delta.sources.DeltaSQLConf -import org.apache.spark.sql.delta.util.DeltaFileOperations -import org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig -import org.apache.spark.sql.functions.{col, expr, when} -import org.apache.spark.util.{Clock, SerializableConfiguration, SystemClock} - -// scalastyle:off import.ordering.noEmptyLine -import org.apache.hadoop.fs.Path - -import java.util.Date -import java.util.concurrent.TimeUnit - -/** - * Vacuums the table by clearing all untracked files and folders within this table. First lists all - * the files and directories in the table, and gets the relative paths with respect to the base of - * the table. Then it gets the list of all tracked files for this table, which may or may not be - * within the table base path, and gets the relative paths of all the tracked files with respect to - * the base of the table. Files outside of the table path will be ignored. Then we take a diff of - * the files and delete directories that were already empty, and all files that are within the table - * that are no longer tracked. - */ -object VacuumCommand extends VacuumCommandImpl with Serializable { - - // --- modified start - case class FileNameAndSize(path: String, length: Long, isDir: Boolean = false) - // --- modified end - - /** - * Additional check on retention duration to prevent people from shooting themselves in the foot. - */ - protected def checkRetentionPeriodSafety( - spark: SparkSession, - retentionMs: Option[Long], - configuredRetention: Long): Unit = { - require(retentionMs.forall(_ >= 0), "Retention for Vacuum can't be less than 0.") - val checkEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RETENTION_CHECK_ENABLED) - val retentionSafe = retentionMs.forall(_ >= configuredRetention) - var configuredRetentionHours = TimeUnit.MILLISECONDS.toHours(configuredRetention) - if (TimeUnit.HOURS.toMillis(configuredRetentionHours) < configuredRetention) { - configuredRetentionHours += 1 - } - require( - !checkEnabled || retentionSafe, - s"""Are you sure you would like to vacuum files with such a low retention period? If you have - |writers that are currently writing to this table, there is a risk that you may corrupt the - |state of your Delta table. - | - |If you are certain that there are no operations being performed on this table, such as - |insert/upsert/delete/optimize, then you may turn off this check by setting: - |spark.databricks.delta.retentionDurationCheck.enabled = false - | - |If you are not sure, please use a value not less than "$configuredRetentionHours hours". - """.stripMargin - ) - } - - /** - * Clears all untracked files and folders within this table. First lists all the files and - * directories in the table, and gets the relative paths with respect to the base of the table. - * Then it gets the list of all tracked files for this table, which may or may not be within the - * table base path, and gets the relative paths of all the tracked files with respect to the base - * of the table. Files outside of the table path will be ignored. Then we take a diff of the files - * and delete directories that were already empty, and all files that are within the table that - * are no longer tracked. - * - * @param dryRun - * If set to true, no files will be deleted. Instead, we will list all files and directories - * that will be cleared. - * @param retentionHours - * An optional parameter to override the default Delta tombstone retention period - * @return - * A Dataset containing the paths of the files/folders to delete in dryRun mode. Otherwise - * returns the base path of the table. - */ - def gc( - spark: SparkSession, - deltaLog: DeltaLog, - dryRun: Boolean = true, - retentionHours: Option[Double] = None, - clock: Clock = new SystemClock): DataFrame = { - recordDeltaOperation(deltaLog, "delta.gc") { - - val path = deltaLog.dataPath - val deltaHadoopConf = deltaLog.newDeltaHadoopConf() - val fs = path.getFileSystem(deltaHadoopConf) - - import spark.implicits._ - - val snapshot = deltaLog.update() - - require( - snapshot.version >= 0, - "No state defined for this table. Is this really " + - "a Delta table? Refusing to garbage collect.") - - // --- modified start - val isMergeTreeFormat = ClickHouseConfig - .isMergeTreeFormatEngine(deltaLog.snapshot.metadata.configuration) - // --- modified end - - val retentionMillis = retentionHours.map(h => TimeUnit.HOURS.toMillis(math.round(h))) - checkRetentionPeriodSafety(spark, retentionMillis, deltaLog.tombstoneRetentionMillis) - - val deleteBeforeTimestamp = retentionMillis - .map(millis => clock.getTimeMillis() - millis) - .getOrElse(deltaLog.minFileRetentionTimestamp) - logInfo( - s"Starting garbage collection (dryRun = $dryRun) of untracked files older than " + - s"${new Date(deleteBeforeTimestamp).toString} in $path") - val hadoopConf = spark.sparkContext.broadcast(new SerializableConfiguration(deltaHadoopConf)) - val basePath = fs.makeQualified(path).toString - var isBloomFiltered = false - val parallelDeleteEnabled = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_ENABLED) - val parallelDeletePartitions = - spark.sessionState.conf - .getConf(DeltaSQLConf.DELTA_VACUUM_PARALLEL_DELETE_PARALLELISM) - .getOrElse(spark.sessionState.conf.numShufflePartitions) - val relativizeIgnoreError = - spark.sessionState.conf.getConf(DeltaSQLConf.DELTA_VACUUM_RELATIVIZE_IGNORE_ERROR) - - // --- modified start - val originalEnabledGluten = - spark.sparkContext.getLocalProperty(GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY) - // gluten can not support vacuum command - spark.sparkContext.setLocalProperty( - GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, - "false") - // --- modified end - - val validFiles = snapshot.stateDS - .mapPartitions { - actions => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - actions.flatMap { - _.unwrap match { - case tombstone: RemoveFile if tombstone.delTimestamp < deleteBeforeTimestamp => - Nil - case fa: FileAction => - val filePath = stringToPath(fa.path) - val validFileOpt = if (filePath.isAbsolute) { - val maybeRelative = - DeltaFileOperations.tryRelativizePath( - fs, - reservoirBase, - filePath, - relativizeIgnoreError) - if (maybeRelative.isAbsolute) { - // This file lives outside the directory of the table - None - } else { - Option(pathToString(maybeRelative)) - } - } else { - Option(pathToString(filePath)) - } - validFileOpt.toSeq.flatMap { - f => - // paths are relative so provide '/' as the basePath. - allValidFiles(f, isBloomFiltered).flatMap { - file => - val dirs = getAllSubdirs("/", file, fs) - dirs ++ Iterator(file) - } - } - case _ => Nil - } - } - } - .toDF("path") - - val partitionColumns = snapshot.metadata.partitionSchema.fieldNames - val parallelism = spark.sessionState.conf.parallelPartitionDiscoveryParallelism - - val allFilesAndDirs = DeltaFileOperations - .recursiveListDirs( - spark, - Seq(basePath), - hadoopConf, - hiddenFileNameFilter = DeltaTableUtils.isHiddenDirectory(partitionColumns, _), - fileListingParallelism = Option(parallelism) - ) - .groupByKey(x => x.path) - .mapGroups { - (k, v) => - val duplicates = v.toSeq - // of all the duplicates we can return the newest file. - duplicates.maxBy(_.modificationTime) - } - - try { - allFilesAndDirs.cache() - - implicit val fileNameAndSizeEncoder = org.apache.spark.sql.Encoders.product[FileNameAndSize] - - val dirCounts = allFilesAndDirs.where('isDir).count() + 1 // +1 for the base path - - // The logic below is as follows: - // 1. We take all the files and directories listed in our reservoir - // 2. We filter all files older than our tombstone retention period and directories - // 3. We get the subdirectories of all files so that we can find non-empty directories - // 4. We groupBy each path, and count to get how many files are in each sub-directory - // 5. We subtract all the valid files and tombstones in our state - // 6. We filter all paths with a count of 1, which will correspond to files not in the - // state, and empty directories. We can safely delete all of these - // --- modified start - val diff = if (isMergeTreeFormat) { - val diff_temp = allFilesAndDirs - .where('modificationTime < deleteBeforeTimestamp || 'isDir) - .mapPartitions { - fileStatusIterator => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - fileStatusIterator.flatMap { - fileStatus => - if (fileStatus.isDir) { - implicit val fileNameAndSizeEncoder = - org.apache.spark.sql.Encoders.product[FileNameAndSize] - Iterator.single( - FileNameAndSize( - relativize(fileStatus.getPath, fs, reservoirBase, isDir = true), - 0, - true) - ) - } else { - val dirs = getAllSubdirs(basePath, fileStatus.path, fs) - val dirsWithSlash = dirs.map { - p => - FileNameAndSize( - relativize(new Path(p), fs, reservoirBase, isDir = true), - 0, - true) - } - dirsWithSlash ++ Iterator( - FileNameAndSize( - relativize(new Path(fileStatus.path), fs, reservoirBase, isDir = false), - 0, - false)) - } - } - } - .withColumn( - "dir", - when(col("isDir"), col("path")) - .otherwise(expr("substring_index(path, '/',size(split(path, '/')) -1)"))) - .groupBy(col("path"), col("dir")) - .count() - - diff_temp - .join(validFiles, diff_temp("dir") === validFiles("path"), "leftanti") - .where('count === 1) - .select('path) - .as[String] - .map { - relativePath => - assert( - !stringToPath(relativePath).isAbsolute, - "Shouldn't have any absolute paths for deletion here.") - pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) - } - } else { - allFilesAndDirs - .where('modificationTime < deleteBeforeTimestamp || 'isDir) - .mapPartitions { - fileStatusIterator => - val reservoirBase = new Path(basePath) - val fs = reservoirBase.getFileSystem(hadoopConf.value.value) - fileStatusIterator.flatMap { - fileStatus => - if (fileStatus.isDir) { - Iterator.single( - relativize(fileStatus.getPath, fs, reservoirBase, isDir = true)) - } else { - val dirs = getAllSubdirs(basePath, fileStatus.path, fs) - val dirsWithSlash = dirs.map { - p => relativize(new Path(p), fs, reservoirBase, isDir = true) - } - dirsWithSlash ++ Iterator( - relativize(new Path(fileStatus.path), fs, reservoirBase, isDir = false)) - } - } - } - .groupBy($"value".as('path)) - .count() - .join(validFiles, Seq("path"), "leftanti") - .where('count === 1) - .select('path) - .as[String] - .map { - relativePath => - assert( - !stringToPath(relativePath).isAbsolute, - "Shouldn't have any absolute paths for deletion here.") - pathToString(DeltaFileOperations.absolutePath(basePath, relativePath)) - } - } - // --- modified end - - if (dryRun) { - val numFiles = diff.count() - val stats = DeltaVacuumStats( - isDryRun = true, - specifiedRetentionMillis = retentionMillis, - defaultRetentionMillis = deltaLog.tombstoneRetentionMillis, - minRetainedTimestamp = deleteBeforeTimestamp, - dirsPresentBeforeDelete = dirCounts, - objectsDeleted = numFiles - ) - - recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) - logConsole( - s"Found $numFiles files and directories in a total of " + - s"$dirCounts directories that are safe to delete.") - - return diff.map(f => stringToPath(f).toString).toDF("path") - } - logVacuumStart( - spark, - deltaLog, - path, - diff, - retentionMillis, - deltaLog.tombstoneRetentionMillis) - - val filesDeleted = - try { - delete( - diff, - spark, - basePath, - hadoopConf, - parallelDeleteEnabled, - parallelDeletePartitions) - } catch { - case t: Throwable => - logVacuumEnd(deltaLog, spark, path) - throw t - } - val stats = DeltaVacuumStats( - isDryRun = false, - specifiedRetentionMillis = retentionMillis, - defaultRetentionMillis = deltaLog.tombstoneRetentionMillis, - minRetainedTimestamp = deleteBeforeTimestamp, - dirsPresentBeforeDelete = dirCounts, - objectsDeleted = filesDeleted - ) - recordDeltaEvent(deltaLog, "delta.gc.stats", data = stats) - logVacuumEnd(deltaLog, spark, path, Some(filesDeleted), Some(dirCounts)) - - spark.createDataset(Seq(basePath)).toDF("path") - } finally { - allFilesAndDirs.unpersist() - - // --- modified start - if (originalEnabledGluten != null) { - spark.sparkContext.setLocalProperty( - GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, - originalEnabledGluten) - } else { - spark.sparkContext.setLocalProperty( - GlutenSessionExtensions.GLUTEN_ENABLE_FOR_THREAD_KEY, - "true") - } - // --- modified end - } - } - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala deleted file mode 100644 index 99360ed0f18..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/files/MergeTreeDelayedCommitProtocol.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.files - -class MergeTreeDelayedCommitProtocol( - val outputPath: String, - randomPrefixLength: Option[Int], - val database: String, - val tableName: String) - extends DelayedCommitProtocol("delta-mergetree", outputPath, randomPrefixLength) - with MergeTreeFileCommitProtocol {} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala deleted file mode 100644 index a360fa8d729..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/delta/rules/CHOptimizeMetadataOnlyDeltaQuery.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.delta.rules - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.delta.metering.DeltaLogging - -class CHOptimizeMetadataOnlyDeltaQuery(protected val spark: SparkSession) - extends Rule[LogicalPlan] - with DeltaLogging { - - // For Delta 2.0, it can not support to optimize query with the metadata - override def apply(plan: LogicalPlan): LogicalPlan = plan -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala deleted file mode 100644 index e949bebf236..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/CHDeltaColumnarWrite.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution - -import org.apache.gluten.exception.GlutenNotSupportException - -import org.apache.spark.internal.io.FileCommitProtocol -import org.apache.spark.sql.execution.datasources.WriteJobDescription - -object CHDeltaColumnarWrite { - def apply( - jobTrackerID: String, - description: WriteJobDescription, - committer: FileCommitProtocol): CHColumnarWrite[FileCommitProtocol] = - throw new GlutenNotSupportException("Delta Native is not supported in Spark 3.2") -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala deleted file mode 100644 index 8c1062f4c7b..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseDataSource.scala +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse - -import org.apache.spark.sql._ -import org.apache.spark.sql.catalyst.util.CaseInsensitiveMap -import org.apache.spark.sql.connector.catalog.Table -import org.apache.spark.sql.connector.expressions.Transform -import org.apache.spark.sql.delta._ -import org.apache.spark.sql.delta.catalog.ClickHouseTableV2 -import org.apache.spark.sql.delta.commands.WriteIntoDelta -import org.apache.spark.sql.delta.commands.cdc.CDCReader -import org.apache.spark.sql.delta.sources.{DeltaDataSource, DeltaSourceUtils, DeltaSQLConf} -import org.apache.spark.sql.sources.BaseRelation -import org.apache.spark.sql.types.StructType -import org.apache.spark.sql.util.CaseInsensitiveStringMap - -import org.apache.hadoop.fs.Path - -import scala.collection.JavaConverters._ -import scala.collection.mutable - -/** A DataSource V1 for integrating Delta into Spark SQL batch and Streaming APIs. */ -class ClickHouseDataSource extends DeltaDataSource { - - override def shortName(): String = { - ClickHouseConfig.NAME - } - - override def getTable( - schema: StructType, - partitioning: Array[Transform], - properties: java.util.Map[String, String]): Table = { - val options = new CaseInsensitiveStringMap(properties) - val path = options.get("path") - if (path == null) throw DeltaErrors.pathNotSpecifiedException - new ClickHouseTableV2( - SparkSession.active, - new Path(path), - options = properties.asScala.toMap, - clickhouseExtensionOptions = ClickHouseConfig - .createMergeTreeConfigurations( - ClickHouseConfig - .getMergeTreeConfigurations(properties) - .asJava) - ) - } - - override def createRelation( - sqlContext: SQLContext, - mode: SaveMode, - parameters: Map[String, String], - data: DataFrame): BaseRelation = { - val path = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) - val partitionColumns = parameters - .get(DeltaSourceUtils.PARTITIONING_COLUMNS_KEY) - .map(DeltaDataSource.decodePartitioningColumns) - .getOrElse(Nil) - - val deltaLog = DeltaLog.forTable(sqlContext.sparkSession, path, parameters) - // need to use the latest snapshot - val configs = if (deltaLog.update().version < 0) { - // when creating table, save the clickhouse config to the delta metadata - val clickHouseTableV2 = ClickHouseTableV2.getTable(deltaLog) - clickHouseTableV2.properties().asScala.toMap ++ DeltaConfigs - .validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) - } else { - DeltaConfigs.validateConfigurations(parameters.filterKeys(_.startsWith("delta.")).toMap) - } - WriteIntoDelta( - deltaLog = deltaLog, - mode = mode, - new DeltaOptions(parameters, sqlContext.sparkSession.sessionState.conf), - partitionColumns = partitionColumns, - configuration = configs, - data = data - ).run(sqlContext.sparkSession) - - deltaLog.createRelation() - } - - override def createRelation( - sqlContext: SQLContext, - parameters: Map[String, String]): BaseRelation = { - recordFrameProfile("Delta", "DeltaDataSource.createRelation") { - val maybePath = parameters.getOrElse("path", throw DeltaErrors.pathNotSpecifiedException) - - // Log any invalid options that are being passed in - DeltaOptions.verifyOptions(CaseInsensitiveMap(parameters)) - - val timeTravelByParams = DeltaDataSource.getTimeTravelVersion(parameters) - var cdcOptions: mutable.Map[String, String] = mutable.Map.empty - val caseInsensitiveParams = new CaseInsensitiveStringMap(parameters.asJava) - if (CDCReader.isCDCRead(caseInsensitiveParams)) { - cdcOptions = mutable.Map[String, String](DeltaDataSource.CDC_ENABLED_KEY -> "true") - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_VERSION_KEY)) { - cdcOptions(DeltaDataSource.CDC_START_VERSION_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_START_VERSION_KEY) - } - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_START_TIMESTAMP_KEY)) { - cdcOptions(DeltaDataSource.CDC_START_TIMESTAMP_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_START_TIMESTAMP_KEY) - } - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_VERSION_KEY)) { - cdcOptions(DeltaDataSource.CDC_END_VERSION_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_END_VERSION_KEY) - } - if (caseInsensitiveParams.containsKey(DeltaDataSource.CDC_END_TIMESTAMP_KEY)) { - cdcOptions(DeltaDataSource.CDC_END_TIMESTAMP_KEY) = - caseInsensitiveParams.get(DeltaDataSource.CDC_END_TIMESTAMP_KEY) - } - } - val dfOptions: Map[String, String] = - if ( - sqlContext.sparkSession.sessionState.conf.getConf( - DeltaSQLConf.LOAD_FILE_SYSTEM_CONFIGS_FROM_DATAFRAME_OPTIONS) - ) { - parameters - } else { - Map.empty - } - (new ClickHouseTableV2( - sqlContext.sparkSession, - new Path(maybePath), - timeTravelOpt = timeTravelByParams, - options = dfOptions, - cdcOptions = new CaseInsensitiveStringMap(cdcOptions.asJava) - )).toBaseRelation - } - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala deleted file mode 100644 index 47b2ae2bd1a..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/ClickHouseSparkCatalog.scala +++ /dev/null @@ -1,661 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse - -import org.apache.spark.sql.{AnalysisException, DataFrame, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.analysis.{NoSuchDatabaseException, NoSuchNamespaceException, NoSuchTableException} -import org.apache.spark.sql.catalyst.catalog._ -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.connector.catalog._ -import org.apache.spark.sql.connector.catalog.TableCapability.V1_BATCH_WRITE -import org.apache.spark.sql.connector.expressions.Transform -import org.apache.spark.sql.connector.write.{LogicalWriteInfo, V1Write, WriteBuilder} -import org.apache.spark.sql.delta.{DeltaConfigs, DeltaErrors, DeltaLog, DeltaOptions, DeltaTableUtils} -import org.apache.spark.sql.delta.DeltaTableIdentifier.gluePermissionError -import org.apache.spark.sql.delta.catalog.{ClickHouseTableV2, DeltaTableV2, TempClickHouseTableV2} -import org.apache.spark.sql.delta.commands.{CreateDeltaTableCommand, TableCreationModes, WriteIntoDelta} -import org.apache.spark.sql.delta.metering.DeltaLogging -import org.apache.spark.sql.delta.sources.{DeltaSourceUtils, DeltaSQLConf} -import org.apache.spark.sql.execution.datasources.{DataSource, PartitioningUtils} -import org.apache.spark.sql.execution.datasources.v2.clickhouse.utils.CHDataSourceUtils -import org.apache.spark.sql.execution.datasources.v2.utils.CatalogUtil -import org.apache.spark.sql.sources.InsertableRelation -import org.apache.spark.sql.types.StructType - -import org.apache.hadoop.fs.Path - -import java.util -import java.util.Locale - -import scala.collection.JavaConverters._ - -class ClickHouseSparkCatalog - extends DelegatingCatalogExtension - with StagingTableCatalog - with SupportsPathIdentifier - with DeltaLogging { - - val spark = SparkSession.active - - private def createCatalogTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String] - ): Table = { - super.createTable(ident, schema, partitions, properties) - } - - override def createTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): Table = { - if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { - createClickHouseTable( - ident, - schema, - partitions, - properties, - Map.empty, - sourceQuery = None, - TableCreationModes.Create) - } else if (DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties))) { - createDeltaTable( - ident, - schema, - partitions, - properties, - Map.empty, - sourceQuery = None, - TableCreationModes.Create - ) - } else { - createCatalogTable(ident, schema, partitions, properties) - } - } - - /** - * Creates a ClickHouse table - * - * @param ident - * The identifier of the table - * @param schema - * The schema of the table - * @param partitions - * The partition transforms for the table - * @param allTableProperties - * The table properties that configure the behavior of the table or provide information about - * the table - * @param writeOptions - * Options specific to the write during table creation or replacement - * @param sourceQuery - * A query if this CREATE request came from a CTAS or RTAS - * @param operation - * The specific table creation mode, whether this is a Create/Replace/Create or Replace - */ - private def createClickHouseTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - allTableProperties: util.Map[String, String], - writeOptions: Map[String, String], - sourceQuery: Option[DataFrame], - operation: TableCreationModes.CreationMode): Table = { - val (partitionColumns, maybeBucketSpec) = - CatalogUtil.convertPartitionTransforms(partitions) - var newSchema = schema - var newPartitionColumns = partitionColumns - var newBucketSpec = maybeBucketSpec - - // Delta does not support bucket feature, so save the bucket infos into properties if exists. - val tableProperties = - ClickHouseConfig.createMergeTreeConfigurations(allTableProperties, newBucketSpec) - - val isByPath = isPathIdentifier(ident) - val location = if (isByPath) { - Option(ident.name()) - } else { - Option(allTableProperties.get("location")) - } - val locUriOpt = location.map(CatalogUtils.stringToURI) - val storage = DataSource - .buildStorageFormatFromOptions(writeOptions) - .copy(locationUri = locUriOpt) - val tableType = - if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED - val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) - } - val existingTableOpt = getExistingTableIfExists(id) - val loc = new Path(locUriOpt.getOrElse(spark.sessionState.catalog.defaultTablePath(id))) - val commentOpt = Option(allTableProperties.get("comment")) - - val tableDesc = new CatalogTable( - identifier = id, - tableType = tableType, - storage = storage, - schema = newSchema, - provider = Some(ClickHouseConfig.ALT_NAME), - partitionColumnNames = newPartitionColumns, - bucketSpec = newBucketSpec, - properties = tableProperties, - comment = commentOpt - ) - - val withDb = verifyTableAndSolidify(tableDesc, None, true) - - val writer = sourceQuery.map { - df => - WriteIntoDelta( - DeltaLog.forTable(spark, loc), - operation.mode, - new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), - withDb.partitionColumnNames, - withDb.properties ++ commentOpt.map("comment" -> _), - df, - schemaInCatalog = if (newSchema != schema) Some(newSchema) else None - ) - } - try { - ClickHouseTableV2.temporalThreadLocalCHTable.set( - new TempClickHouseTableV2(spark, Some(withDb))) - - CreateDeltaTableCommand( - withDb, - existingTableOpt, - operation.mode, - writer, - operation = operation, - tableByPath = isByPath).run(spark) - } finally { - ClickHouseTableV2.temporalThreadLocalCHTable.remove() - } - - logInfo(s"create table ${ident.toString} successfully.") - loadTable(ident) - } - - /** - * Creates a Delta table - * - * @param ident - * The identifier of the table - * @param schema - * The schema of the table - * @param partitions - * The partition transforms for the table - * @param allTableProperties - * The table properties that configure the behavior of the table or provide information about - * the table - * @param writeOptions - * Options specific to the write during table creation or replacement - * @param sourceQuery - * A query if this CREATE request came from a CTAS or RTAS - * @param operation - * The specific table creation mode, whether this is a Create/Replace/Create or Replace - */ - private def createDeltaTable( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - allTableProperties: util.Map[String, String], - writeOptions: Map[String, String], - sourceQuery: Option[DataFrame], - operation: TableCreationModes.CreationMode - ): Table = { - // These two keys are tableProperties in data source v2 but not in v1, so we have to filter - // them out. Otherwise property consistency checks will fail. - val tableProperties = allTableProperties.asScala.filterKeys { - case TableCatalog.PROP_LOCATION => false - case TableCatalog.PROP_PROVIDER => false - case TableCatalog.PROP_COMMENT => false - case TableCatalog.PROP_OWNER => false - case TableCatalog.PROP_EXTERNAL => false - case "path" => false - case _ => true - }.toMap - val (partitionColumns, maybeBucketSpec) = - CatalogUtil.convertPartitionTransforms(partitions) - var newSchema = schema - var newPartitionColumns = partitionColumns - var newBucketSpec = maybeBucketSpec - val conf = spark.sessionState.conf - - val isByPath = isPathIdentifier(ident) - if ( - isByPath && !conf.getConf(DeltaSQLConf.DELTA_LEGACY_ALLOW_AMBIGUOUS_PATHS) - && allTableProperties.containsKey("location") - // The location property can be qualified and different from the path in the identifier, so - // we check `endsWith` here. - && Option(allTableProperties.get("location")).exists(!_.endsWith(ident.name())) - ) { - throw DeltaErrors.ambiguousPathsInCreateTableException( - ident.name(), - allTableProperties.get("location")) - } - val location = if (isByPath) { - Option(ident.name()) - } else { - Option(allTableProperties.get("location")) - } - val id = { - TableIdentifier(ident.name(), ident.namespace().lastOption) - } - var locUriOpt = location.map(CatalogUtils.stringToURI) - val existingTableOpt = getExistingTableIfExists(id) - val loc = locUriOpt - .orElse(existingTableOpt.flatMap(_.storage.locationUri)) - .getOrElse(spark.sessionState.catalog.defaultTablePath(id)) - val storage = DataSource - .buildStorageFormatFromOptions(writeOptions) - .copy(locationUri = Option(loc)) - val tableType = - if (location.isDefined) CatalogTableType.EXTERNAL else CatalogTableType.MANAGED - val commentOpt = Option(allTableProperties.get("comment")) - - var tableDesc = new CatalogTable( - identifier = id, - tableType = tableType, - storage = storage, - schema = newSchema, - provider = Some(DeltaSourceUtils.ALT_NAME), - partitionColumnNames = newPartitionColumns, - bucketSpec = newBucketSpec, - properties = tableProperties, - comment = commentOpt - ) - - val withDb = verifyTableAndSolidify(tableDesc, None) - - val writer = sourceQuery.map { - df => - WriteIntoDelta( - DeltaLog.forTable(spark, new Path(loc)), - operation.mode, - new DeltaOptions(withDb.storage.properties, spark.sessionState.conf), - withDb.partitionColumnNames, - withDb.properties ++ commentOpt.map("comment" -> _), - df, - schemaInCatalog = if (newSchema != schema) Some(newSchema) else None - ) - } - - CreateDeltaTableCommand( - withDb, - existingTableOpt, - operation.mode, - writer, - operation, - tableByPath = isByPath).run(spark) - - loadTable(ident) - } - - /** Performs checks on the parameters provided for table creation for a ClickHouse table. */ - private def verifyTableAndSolidify( - tableDesc: CatalogTable, - query: Option[LogicalPlan], - isMergeTree: Boolean = false): CatalogTable = { - - if (!isMergeTree && tableDesc.bucketSpec.isDefined) { - throw DeltaErrors.operationNotSupportedException("Bucketing", tableDesc.identifier) - } - - val schema = query - .map { - plan => - assert(tableDesc.schema.isEmpty, "Can't specify table schema in CTAS.") - plan.schema.asNullable - } - .getOrElse(tableDesc.schema) - - PartitioningUtils.validatePartitionColumn( - schema, - tableDesc.partitionColumnNames, - caseSensitive = false - ) // Delta is case insensitive - - val validatedConfigurations = if (isMergeTree) { - tableDesc.properties - } else { - DeltaConfigs.validateConfigurations(tableDesc.properties) - } - - val db = tableDesc.identifier.database.getOrElse(catalog.getCurrentDatabase) - val tableIdentWithDB = tableDesc.identifier.copy(database = Some(db)) - tableDesc.copy( - identifier = tableIdentWithDB, - schema = schema, - properties = validatedConfigurations) - } - - /** Checks if a table already exists for the provided identifier. */ - def getExistingTableIfExists(table: TableIdentifier): Option[CatalogTable] = { - // If this is a path identifier, we cannot return an existing CatalogTable. The Create command - // will check the file system itself - if (isPathIdentifier(table)) return None - val tableExists = catalog.tableExists(table) - if (tableExists) { - val oldTable = catalog.getTableMetadata(table) - if (oldTable.tableType == CatalogTableType.VIEW) { - throw new AnalysisException(s"$table is a view. You may not write data into a view.") - } - if ( - !DeltaSourceUtils.isDeltaTable(oldTable.provider) && - !CHDataSourceUtils.isClickHouseTable(oldTable.provider) - ) { - throw DeltaErrors.notADeltaTable(table.table) - } - Some(oldTable) - } else { - None - } - } - - private def getProvider(properties: util.Map[String, String]): String = { - Option(properties.get("provider")).getOrElse(ClickHouseConfig.NAME) - } - - override def loadTable(ident: Identifier): Table = { - try { - super.loadTable(ident) match { - case v1: V1Table if CHDataSourceUtils.isClickHouseTable(v1.catalogTable) => - new ClickHouseTableV2( - spark, - new Path(v1.catalogTable.location), - catalogTable = Some(v1.catalogTable), - tableIdentifier = Some(ident.toString)) - case v1: V1Table if DeltaTableUtils.isDeltaTable(v1.catalogTable) => - DeltaTableV2( - spark, - new Path(v1.catalogTable.location), - catalogTable = Some(v1.catalogTable), - tableIdentifier = Some(ident.toString)) - case o => - o - } - } catch { - case _: NoSuchDatabaseException | _: NoSuchNamespaceException | _: NoSuchTableException - if isPathIdentifier(ident) => - newDeltaPathTable(ident) - case e: AnalysisException if gluePermissionError(e) && isPathIdentifier(ident) => - logWarning( - "Received an access denied error from Glue. Assuming this " + - s"identifier ($ident) is path based.", - e) - newDeltaPathTable(ident) - } - } - - private def newDeltaPathTable(ident: Identifier): DeltaTableV2 = { - if (hasClickHouseNamespace(ident)) { - new ClickHouseTableV2(spark, new Path(ident.name())) - } else { - DeltaTableV2(spark, new Path(ident.name())) - } - } - - /** support to delete mergetree data from the external table */ - override def purgeTable(ident: Identifier): Boolean = { - try { - loadTable(ident) match { - case t: ClickHouseTableV2 => - val tableType = t.properties().getOrDefault("Type", "") - // file-based or external table - val isExternal = tableType.isEmpty || tableType.equalsIgnoreCase("external") - val tablePath = t.rootPath - // first delete the table metadata - val deletedTable = super.dropTable(ident) - if (deletedTable && isExternal) { - val fs = tablePath.getFileSystem(spark.sessionState.newHadoopConf()) - // delete all data if there is a external table - fs.delete(tablePath, true) - } - true - case _ => super.purgeTable(ident) - } - } catch { - case _: Exception => - false - } - } - - override def stageReplace( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): StagedTable = - recordFrameProfile("DeltaCatalog", "stageReplace") { - if ( - CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || - DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) - ) { - new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Replace) - } else { - super.dropTable(ident) - val table = createCatalogTable(ident, schema, partitions, properties) - BestEffortStagedTable(ident, table, this) - } - } - - override def stageCreateOrReplace( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): StagedTable = - recordFrameProfile("DeltaCatalog", "stageCreateOrReplace") { - if ( - CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || - DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) - ) { - new StagedDeltaTableV2( - ident, - schema, - partitions, - properties, - TableCreationModes.CreateOrReplace) - } else { - try super.dropTable(ident) - catch { - case _: NoSuchDatabaseException => // this is fine - case _: NoSuchTableException => // this is fine - } - val table = createCatalogTable(ident, schema, partitions, properties) - BestEffortStagedTable(ident, table, this) - } - } - - override def stageCreate( - ident: Identifier, - schema: StructType, - partitions: Array[Transform], - properties: util.Map[String, String]): StagedTable = - recordFrameProfile("DeltaCatalog", "stageCreate") { - if ( - CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties)) || - DeltaSourceUtils.isDeltaDataSourceName(getProvider(properties)) - ) { - new StagedDeltaTableV2(ident, schema, partitions, properties, TableCreationModes.Create) - } else { - val table = createCatalogTable(ident, schema, partitions, properties) - BestEffortStagedTable(ident, table, this) - } - } - - /** - * A staged delta table, which creates a HiveMetaStore entry and appends data if this was a - * CTAS/RTAS command. We have a ugly way of using this API right now, but it's the best way to - * maintain old behavior compatibility between Databricks Runtime and OSS Delta Lake. - */ - private class StagedDeltaTableV2( - ident: Identifier, - override val schema: StructType, - val partitions: Array[Transform], - override val properties: util.Map[String, String], - operation: TableCreationModes.CreationMode) - extends StagedTable - with SupportsWrite { - - private var asSelectQuery: Option[DataFrame] = None - private var writeOptions: Map[String, String] = Map.empty - - override def commitStagedChanges(): Unit = - recordFrameProfile("DeltaCatalog", "commitStagedChanges") { - val conf = spark.sessionState.conf - val props = new util.HashMap[String, String]() - // Options passed in through the SQL API will show up both with an "option." prefix and - // without in Spark 3.1, so we need to remove those from the properties - val optionsThroughProperties = properties.asScala.collect { - case (k, _) if k.startsWith("option.") => k.stripPrefix("option.") - }.toSet - val sqlWriteOptions = new util.HashMap[String, String]() - properties.asScala.foreach { - case (k, v) => - if (!k.startsWith("option.") && !optionsThroughProperties.contains(k)) { - // Do not add to properties - props.put(k, v) - } else if (optionsThroughProperties.contains(k)) { - sqlWriteOptions.put(k, v) - } - } - if (writeOptions.isEmpty && !sqlWriteOptions.isEmpty) { - writeOptions = sqlWriteOptions.asScala.toMap - } - if (conf.getConf(DeltaSQLConf.DELTA_LEGACY_STORE_WRITER_OPTIONS_AS_PROPS)) { - // Legacy behavior - writeOptions.foreach { case (k, v) => props.put(k, v) } - } else { - writeOptions.foreach { - case (k, v) => - // Continue putting in Delta prefixed options to avoid breaking workloads - if (k.toLowerCase(Locale.ROOT).startsWith("delta.")) { - props.put(k, v) - } - } - } - if (CHDataSourceUtils.isClickHouseDataSourceName(getProvider(properties))) { - createClickHouseTable( - ident, - schema, - partitions, - props, - writeOptions, - asSelectQuery, - operation) - } else { - createDeltaTable(ident, schema, partitions, props, writeOptions, asSelectQuery, operation) - } - } - - override def name(): String = ident.name() - - override def abortStagedChanges(): Unit = {} - - override def capabilities(): util.Set[TableCapability] = Set(V1_BATCH_WRITE).asJava - - override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = { - writeOptions = info.options.asCaseSensitiveMap().asScala.toMap - new DeltaV1WriteBuilder - } - - /* - * WriteBuilder for creating a Delta table. - */ - private class DeltaV1WriteBuilder extends WriteBuilder { - override def build(): V1Write = new V1Write { - override def toInsertableRelation(): InsertableRelation = { - new InsertableRelation { - override def insert(data: DataFrame, overwrite: Boolean): Unit = { - asSelectQuery = Option(data) - } - } - } - } - } - } - - private case class BestEffortStagedTable(ident: Identifier, table: Table, catalog: TableCatalog) - extends StagedTable - with SupportsWrite { - override def abortStagedChanges(): Unit = catalog.dropTable(ident) - - override def commitStagedChanges(): Unit = {} - - // Pass through - override def name(): String = table.name() - override def schema(): StructType = table.schema() - override def partitioning(): Array[Transform] = table.partitioning() - override def capabilities(): util.Set[TableCapability] = table.capabilities() - override def properties(): util.Map[String, String] = table.properties() - - override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = table match { - case supportsWrite: SupportsWrite => supportsWrite.newWriteBuilder(info) - case _ => throw DeltaErrors.unsupportedWriteStagedTable(name) - } - } -} - -/** - * A trait for handling table access through clickhouse.`/some/path`. This is a stop-gap solution - * until PathIdentifiers are implemented in Apache Spark. - */ -trait SupportsPathIdentifier extends TableCatalog { - self: ClickHouseSparkCatalog => - - protected lazy val catalog: SessionCatalog = spark.sessionState.catalog - - override def tableExists(ident: Identifier): Boolean = { - if (isPathIdentifier(ident)) { - val path = new Path(ident.name()) - val fs = path.getFileSystem(spark.sessionState.newHadoopConf()) - fs.exists(path) && fs.listStatus(path).nonEmpty - } else { - super.tableExists(ident) - } - } - - protected def isPathIdentifier(ident: Identifier): Boolean = { - // Should be a simple check of a special PathIdentifier class in the future - try { - supportSQLOnFile && (hasClickHouseNamespace(ident) || hasDeltaNamespace(ident)) && - new Path(ident.name()).isAbsolute - } catch { - case _: IllegalArgumentException => false - } - } - - protected def isPathIdentifier(table: CatalogTable): Boolean = { - isPathIdentifier(table.identifier) - } - - protected def isPathIdentifier(tableIdentifier: TableIdentifier): Boolean = { - isPathIdentifier(Identifier.of(tableIdentifier.database.toArray, tableIdentifier.table)) - } - - private def supportSQLOnFile: Boolean = spark.sessionState.conf.runSQLonFile - - protected def hasClickHouseNamespace(ident: Identifier): Boolean = { - ident.namespace().length == 1 && - CHDataSourceUtils.isClickHouseDataSourceName(ident.namespace().head) - } - - protected def hasDeltaNamespace(ident: Identifier): Boolean = { - ident.namespace().length == 1 && DeltaSourceUtils.isDeltaDataSourceName(ident.namespace().head) - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala deleted file mode 100644 index 68869512b00..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/metadata/AddFileTags.scala +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.metadata - -import org.apache.spark.sql.delta.actions.AddFile -import org.apache.spark.sql.delta.util.MergeTreePartitionUtils -import org.apache.spark.sql.execution.datasources.clickhouse.WriteReturnedMetric - -import com.fasterxml.jackson.core.`type`.TypeReference -import com.fasterxml.jackson.databind.ObjectMapper -import org.apache.hadoop.fs.Path - -import java.util.{List => JList} - -import scala.collection.JavaConverters._ - -@SuppressWarnings(Array("io.github.zhztheplayer.scalawarts.InheritFromCaseClass")) -class AddMergeTreeParts( - val database: String, - val table: String, - val engine: String, // default is "MergeTree" - val tablePath: String, // table path - val targetNode: String, // the node which the current part is generated - val name: String, // part name - val uuid: String, - val rows: Long, // row count - override val size: Long, // the size of the part - val dataCompressedBytes: Long, - val dataUncompressedBytes: Long, - override val modificationTime: Long, - val partitionId: String, - val minBlockNumber: Long, - val maxBlockNumber: Long, - val level: Int, - val dataVersion: Long, - val bucketNum: String, - val dirName: String, - override val dataChange: Boolean, - val partition: String = "", - val defaultCompressionCodec: String = "LZ4", - override val stats: String = "", - override val partitionValues: Map[String, String] = Map.empty[String, String], - val partType: String = "Wide", - val active: Int = 1, - val marks: Long = -1L, // mark count - val marksBytes: Long = -1L, - val removeTime: Long = -1L, - val refcount: Int = -1, - val minDate: Int = -1, - val maxDate: Int = -1, - val minTime: Long = -1L, - val maxTime: Long = -1L, - val primaryKeyBytesInMemory: Long = -1L, - val primaryKeyBytesInMemoryAllocated: Long = -1L, - val isFrozen: Int = 0, - val diskName: String = "default", - val hashOfAllFiles: String = "", - val hashOfUncompressedFiles: String = "", - val uncompressedHashOfCompressedFiles: String = "", - val deleteTtlInfoMin: Long = -1L, - val deleteTtlInfoMax: Long = -1L, - val moveTtlInfoExpression: String = "", - val moveTtlInfoMin: Long = -1L, - val moveTtlInfoMax: Long = -1L, - val recompressionTtlInfoExpression: String = "", - val recompressionTtlInfoMin: Long = -1L, - val recompressionTtlInfoMax: Long = -1L, - val groupByTtlInfoExpression: String = "", - val groupByTtlInfoMin: Long = -1L, - val groupByTtlInfoMax: Long = -1L, - val rowsWhereTtlInfoExpression: String = "", - val rowsWhereTtlInfoMin: Long = -1L, - val rowsWhereTtlInfoMax: Long = -1L, - override val tags: Map[String, String] = null) - extends AddFile(name, partitionValues, size, modificationTime, dataChange, stats, tags) { - - def fullPartPath(): String = { - dirName + "/" + name - } -} - -object AddFileTags { - // scalastyle:off argcount - private def partsInfoToAddFile( - database: String, - table: String, - engine: String, - tablePath: String, - targetNode: String, - name: String, - uuid: String, - rows: Long, - bytesOnDisk: Long, - dataCompressedBytes: Long, - dataUncompressedBytes: Long, - modificationTime: Long, - partitionId: String, - minBlockNumber: Long, - maxBlockNumber: Long, - level: Int, - dataVersion: Long, - bucketNum: String, - dirName: String, - dataChange: Boolean, - partition: String = "", - defaultCompressionCodec: String = "LZ4", - stats: String = "", - partitionValues: Map[String, String] = Map.empty[String, String], - marks: Long = -1L): AddFile = { - // scalastyle:on argcount - val tags = Map[String, String]( - "database" -> database, - "table" -> table, - "engine" -> engine, - "path" -> tablePath, - "targetNode" -> targetNode, - "partition" -> partition, - "uuid" -> uuid, - "rows" -> rows.toString, - "bytesOnDisk" -> bytesOnDisk.toString, - "dataCompressedBytes" -> dataCompressedBytes.toString, - "dataUncompressedBytes" -> dataUncompressedBytes.toString, - "modificationTime" -> modificationTime.toString, - "partitionId" -> partitionId, - "minBlockNumber" -> minBlockNumber.toString, - "maxBlockNumber" -> maxBlockNumber.toString, - "level" -> level.toString, - "dataVersion" -> dataVersion.toString, - "defaultCompressionCodec" -> defaultCompressionCodec, - "bucketNum" -> bucketNum, - "dirName" -> dirName, - "marks" -> marks.toString - ) - val mapper: ObjectMapper = new ObjectMapper() - val rootNode = mapper.createObjectNode() - rootNode.put("numRecords", rows) - rootNode.put("minValues", "") - rootNode.put("maxValues", "") - rootNode.put("nullCount", "") - // Add the `stats` into delta meta log - val metricsStats = mapper.writeValueAsString(rootNode) - val uriName = new Path(name).toUri.toString - AddFile(uriName, partitionValues, bytesOnDisk, modificationTime, dataChange, metricsStats, tags) - } - - def addFileToAddMergeTreeParts(addFile: AddFile): AddMergeTreeParts = { - assert(addFile.tags != null && addFile.tags.nonEmpty) - new AddMergeTreeParts( - addFile.tags("database"), - addFile.tags("table"), - addFile.tags("engine"), - addFile.tags("path"), - addFile.tags("targetNode"), - addFile.path, - addFile.tags("uuid"), - addFile.tags("rows").toLong, - addFile.size, - addFile.tags("dataCompressedBytes").toLong, - addFile.tags("dataUncompressedBytes").toLong, - addFile.modificationTime, - addFile.tags("partitionId"), - addFile.tags("minBlockNumber").toLong, - addFile.tags("maxBlockNumber").toLong, - addFile.tags("level").toInt, - addFile.tags("dataVersion").toLong, - addFile.tags("bucketNum"), - addFile.tags("dirName"), - addFile.dataChange, - addFile.tags("partition"), - addFile.tags("defaultCompressionCodec"), - addFile.stats, - addFile.partitionValues, - marks = addFile.tags("marks").toLong, - tags = addFile.tags - ) - } - - def partsMetricsToAddFile( - database: String, - tableName: String, - originPathStr: String, - returnedMetrics: String, - hostName: Seq[String]): Seq[AddFile] = { - - val mapper: ObjectMapper = new ObjectMapper() - val values: JList[WriteReturnedMetric] = - mapper.readValue(returnedMetrics, new TypeReference[JList[WriteReturnedMetric]]() {}) - val path = new Path(originPathStr) - val modificationTime = System.currentTimeMillis() - - values.asScala.map { - value => - val partitionValues = if (value.getPartitionValues.isEmpty) { - Map.empty[String, String] - } else { - MergeTreePartitionUtils.parsePartitions(value.getPartitionValues) - } - - AddFileTags.partsInfoToAddFile( - database, - tableName, - "MergeTree", - path.toUri.getPath, - hostName.map(_.trim).mkString(","), - value.getPartName, - "", - value.getRowCount, - value.getDiskSize, - -1L, - -1L, - modificationTime, - "", - -1L, - -1L, - -1, - -1L, - value.getBucketId, - path.toString, - dataChange = true, - "", - partitionValues = partitionValues, - marks = value.getMarkCount - ) - }.toSeq - } -} diff --git a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala b/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala deleted file mode 100644 index 65d93a920fa..00000000000 --- a/backends-clickhouse/src-delta20/main/scala/org/apache/spark/sql/execution/datasources/v2/clickhouse/source/DeltaMergeTreeFileFormat.scala +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.sql.execution.datasources.v2.clickhouse.source - -import org.apache.spark.sql.delta.{DeltaParquetFileFormat, MergeTreeFileFormat} -import org.apache.spark.sql.delta.actions.Metadata - -class DeltaMergeTreeFileFormat(val metadata: Metadata) - extends DeltaParquetFileFormat(metadata.columnMappingMode, metadata.schema) - with MergeTreeFileFormat { - - override def equals(other: Any): Boolean = { - other match { - case ff: DeltaMergeTreeFileFormat => - ff.columnMappingMode == columnMappingMode && ff.referenceSchema == referenceSchema - case _ => false - } - } - - override def hashCode(): Int = getClass.getCanonicalName.hashCode() -} diff --git a/backends-clickhouse/src-delta20/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala b/backends-clickhouse/src-delta20/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala deleted file mode 100644 index 5d9f761e8a7..00000000000 --- a/backends-clickhouse/src-delta20/test/scala/org/apache/spark/gluten/delta/DeltaStatsUtils.scala +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.spark.gluten.delta - -import org.apache.spark.sql.{DataFrame, SparkSession} - -object DeltaStatsUtils { - - def statsDF( - sparkSession: SparkSession, - deltaJson: String, - schema: String - ): DataFrame = { - throw new IllegalAccessException("Method not used below spark 3.5") - } -} diff --git a/gluten-delta/src-delta20/main/scala/org/apache/gluten/delta/DeltaDeletionVectorScanInfo.scala b/gluten-delta/src-delta20/main/scala/org/apache/gluten/delta/DeltaDeletionVectorScanInfo.scala deleted file mode 100644 index af8e8df7da8..00000000000 --- a/gluten-delta/src-delta20/main/scala/org/apache/gluten/delta/DeltaDeletionVectorScanInfo.scala +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.delta - -import org.apache.gluten.substrait.rel.DeltaLocalFilesNode.DeltaFileReadOptions - -import org.apache.spark.sql.execution.datasources.PartitionedFile - -import java.util.{Map => JMap} - -/** Reading deletion vectors natively requires Delta 3.3+, so there is nothing to materialize. */ -object DeltaDeletionVectorScanInfo { - def normalize(partitionColumnCount: Int, partitionFiles: Seq[PartitionedFile]) - : Option[(Seq[JMap[String, Object]], Seq[DeltaFileReadOptions])] = None -} diff --git a/gluten-delta/src-delta20/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala b/gluten-delta/src-delta20/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala deleted file mode 100644 index ca4665c0d0c..00000000000 --- a/gluten-delta/src-delta20/main/scala/org/apache/gluten/execution/DeltaFilterExecTransformer.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.sql.catalyst.expressions.Expression -import org.apache.spark.sql.execution.SparkPlan - -case class DeltaFilterExecTransformer(condition: Expression, child: SparkPlan) - extends FilterExecTransformerBase(condition, child) { - - override protected def withNewChildInternal(newChild: SparkPlan): DeltaFilterExecTransformer = - copy(child = newChild) -} diff --git a/gluten-delta/src-delta20/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala b/gluten-delta/src-delta20/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala deleted file mode 100644 index 9b720b19c5b..00000000000 --- a/gluten-delta/src-delta20/main/scala/org/apache/gluten/execution/DeltaProjectExecTransformer.scala +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.gluten.execution - -import org.apache.spark.sql.catalyst.expressions.NamedExpression -import org.apache.spark.sql.execution.SparkPlan - -case class DeltaProjectExecTransformer(projectList: Seq[NamedExpression], child: SparkPlan) - extends ProjectExecTransformerBase(projectList, child) { - - override protected def withNewChildInternal(newChild: SparkPlan): DeltaProjectExecTransformer = - copy(child = newChild) -}