Skip to content

Commit

Permalink
NIFI-13934 Removed Hive 3 Catalog from Iceberg Services
Browse files Browse the repository at this point in the history 
- Removed Iceberg Hive Metastore Catalog implementation due Hive 3 end of life declaration
- Removed associated dependency check suppressions no longer needed
  • Loading branch information
@exceptionfactory
exceptionfactory committed Oct 26, 2024
1 parent b6952f1 commit 2db7bfa
Show file tree
Hide file tree
Showing 16 changed files with 0 additions and 2,136 deletions.
65 changes: 0 additions & 65 deletions nifi-dependency-check-maven/suppressions.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,66 +79,6 @@
<packageUrl regex="true">^pkg:maven/org\.apache\.parquet/parquet\-avro@.*$</packageUrl>
<cpe>cpe:/a:avro_project:avro</cpe>
</suppress>
<suppress>
<notes>CVE-2016-5397 applies to Apache Thrift Go not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libthrift@.*$</packageUrl>
<cve>CVE-2016-5397</cve>
</suppress>
<suppress>
<notes>CVE-2019-0210 applies to Apache Thrift Go server not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libthrift@.*$</packageUrl>
<cve>CVE-2019-0210</cve>
</suppress>
<suppress>
<notes>CVE-2018-11798 applies Apache Thrift Node.js not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libthrift@.*$</packageUrl>
<cve>CVE-2018-11798</cve>
</suppress>
<suppress>
<notes>CVE-2019-11939 applies to Thrift Servers in Go not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-11939</cve>
</suppress>
<suppress>
<notes>CVE-2019-3552 applies to Thrift Servers in CPP not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-3552</cve>
</suppress>
<suppress>
<notes>CVE-2019-3553 applies to Thrift Servers in CPP not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-3553</cve>
</suppress>
<suppress>
<notes>CVE-2019-3558 applies to Thrift Servers in Python not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-3558</cve>
</suppress>
<suppress>
<notes>CVE-2019-3564 applies to Thrift Servers in Go not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-3564</cve>
</suppress>
<suppress>
<notes>CVE-2019-3565 applies to Thrift Servers in CPP not Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-3565</cve>
</suppress>
<suppress>
<notes>CVE-2021-24028 applies to Facebook Thrift CPP</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2021-24028</cve>
</suppress>
<suppress>
<notes>CVE-2019-11938 applies to Facebook Thrift Servers</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-11938</cve>
</suppress>
<suppress>
<notes>CVE-2019-3559 applies to Facebook Thrift Servers</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.thrift/libfb303@.*$</packageUrl>
<cve>CVE-2019-3559</cve>
</suppress>
<suppress>
<notes>CVE-2023-37475 applies to Hamba Avro in Go not Apache Avro for Java</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.avro/.*$</packageUrl>
Expand All @@ -149,11 +89,6 @@
<packageUrl regex="true">^pkg:maven/com\.azure/azure\-identity@.*$</packageUrl>
<cve>CVE-2023-36415</cve>
</suppress>
<suppress>
<notes>CVE-2020-13949 applies to Thrift and not to Hive</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.hive.*$</packageUrl>
<cve>CVE-2020-13949</cve>
</suppress>
<suppress>
<notes>Parquet MR vulnerabilities do not apply to other Parquet libraries</notes>
<packageUrl regex="true">^pkg:maven/org\.apache\.parquet/parquet\-(?!mr).*$</packageUrl>
Expand Down
5 changes: 0 additions & 5 deletions nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-common/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,11 +69,6 @@
<artifactId>iceberg-core</artifactId>
<version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-hive-metastore</artifactId>
<version>${iceberg.version}</version>
</dependency>
<dependency>
<groupId>org.apache.iceberg</groupId>
<artifactId>iceberg-data</artifactId>
Expand Down
26 changes: 0 additions & 26 deletions ...ommon/src/main/java/org/apache/nifi/processors/iceberg/catalog/IcebergCatalogFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import org.apache.iceberg.CatalogUtil;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.hadoop.HadoopCatalog;
import org.apache.iceberg.hive.HiveCatalog;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.jdbc.JdbcCatalog;
import org.apache.iceberg.jdbc.JdbcClientPool;
Expand All @@ -37,7 +36,6 @@
import static org.apache.nifi.processors.iceberg.IcebergUtils.getConfigurationFromFiles;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CATALOG_NAME;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.CLIENT_POOL_SERVICE;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.METASTORE_URI;
import static org.apache.nifi.services.iceberg.IcebergCatalogProperty.WAREHOUSE_LOCATION;

public class IcebergCatalogFactory {
Expand All @@ -50,35 +48,11 @@ public IcebergCatalogFactory(IcebergCatalogService catalogService) {

public Catalog create() {
return switch (catalogService.getCatalogType()) {
case HIVE -> initHiveCatalog(catalogService);
case HADOOP -> initHadoopCatalog(catalogService);
case JDBC -> initJdbcCatalog(catalogService);
};
}

private Catalog initHiveCatalog(IcebergCatalogService catalogService) {
HiveCatalog catalog = new HiveCatalog();

if (catalogService.getConfigFilePaths() != null) {
final Configuration configuration = getConfigurationFromFiles(catalogService.getConfigFilePaths());
catalog.setConf(configuration);
}

final Map<IcebergCatalogProperty, Object> catalogProperties = catalogService.getCatalogProperties();
final Map<String, String> properties = new HashMap<>();

if (catalogProperties.containsKey(METASTORE_URI)) {
properties.put(CatalogProperties.URI, (String) catalogProperties.get(METASTORE_URI));
}

if (catalogProperties.containsKey(WAREHOUSE_LOCATION)) {
properties.put(CatalogProperties.WAREHOUSE_LOCATION, (String) catalogProperties.get(WAREHOUSE_LOCATION));
}

catalog.initialize("hive-catalog", properties);
return catalog;
}

private Catalog initHadoopCatalog(IcebergCatalogService catalogService) {
final Map<IcebergCatalogProperty, Object> catalogProperties = catalogService.getCatalogProperties();
final String warehousePath = (String) catalogProperties.get(WAREHOUSE_LOCATION);
Expand Down
205 changes: 0 additions & 205 deletions nifi-extension-bundles/nifi-iceberg-bundle/nifi-iceberg-processors/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,209 +53,10 @@
</dependency>

<!-- External dependencies -->
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-exec</artifactId>
<version>${hive.version}</version>
<classifier>core</classifier>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-web</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hive</groupId>
<artifactId>hive-llap-tez</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite</groupId>
<artifactId>calcite-druid</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.calcite.avatica</groupId>
<artifactId>avatica</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-registry</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
</exclusion>
<exclusion>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk15on</artifactId>
</exclusion>
<exclusion>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.ivy</groupId>
<artifactId>ivy</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.bouncycastle</groupId>
<artifactId>bcprov-jdk18on</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-web</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-mapreduce</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-hadoop2-compat</artifactId>
</exclusion>
<exclusion>
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-api</artifactId>
</exclusion>
<exclusion>
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-core</artifactId>
</exclusion>
<exclusion>
<groupId>co.cask.tephra</groupId>
<artifactId>tephra-hbase-compat-1.0</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop-bundle</artifactId>
</exclusion>
<exclusion>
<groupId>com.tdunning</groupId>
<artifactId>json</artifactId>
</exclusion>
<exclusion>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
</exclusion>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-xml</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-servlet</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-webapp</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-util</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-http</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-rewrite</artifactId>
</exclusion>
</exclusions>
</dependency>
<!-- Include Derby Tools since Derby 10.16 moved the EmbeddedDriver class -->
<dependency>
<groupId>org.apache.derby</groupId>
Expand Down Expand Up @@ -297,12 +98,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-iceberg-test-utils</artifactId>
<version>2.0.0-SNAPSHOT</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.nifi</groupId>
<artifactId>nifi-avro-record-utils</artifactId>
Expand Down
Loading

0 comments on commit 2db7bfa

Please sign in to comment.