-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #110 from HuemulSolutions/develop_2.5
Develop 2.5
- Loading branch information
Showing
14 changed files
with
1,180 additions
and
288 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
474 changes: 474 additions & 0 deletions
474
src/main/resources/Instalacion/huemul_bdg_2.5_minor.sql
Large diffs are not rendered by default.
Oops, something went wrong.
44 changes: 44 additions & 0 deletions
44
src/main/resources/Instalacion/huemul_cluster_setting_2.5.sh
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,44 @@ | ||
| #!/bin/bash | ||
| clear | ||
| echo "Creating HDFS Paths: START" | ||
| hdfs dfs -mkdir /user | ||
| hdfs dfs -mkdir /user/data | ||
| hdfs dfs -mkdir /user/data/production | ||
| hdfs dfs -mkdir /user/data/production/temp | ||
| hdfs dfs -mkdir /user/data/production/raw | ||
| hdfs dfs -mkdir /user/data/production/master | ||
| hdfs dfs -mkdir /user/data/production/dim | ||
| hdfs dfs -mkdir /user/data/production/analytics | ||
| hdfs dfs -mkdir /user/data/production/reporting | ||
| hdfs dfs -mkdir /user/data/production/sandbox | ||
| hdfs dfs -mkdir /user/data/production/dqerror | ||
| hdfs dfs -mkdir /user/data/production/mdm_oldvalue | ||
| hdfs dfs -mkdir /user/data/production/backup | ||
| hdfs dfs -mkdir /user/data/experimental | ||
| hdfs dfs -mkdir /user/data/experimental/temp | ||
| hdfs dfs -mkdir /user/data/experimental/raw | ||
| hdfs dfs -mkdir /user/data/experimental/master | ||
| hdfs dfs -mkdir /user/data/experimental/dim | ||
| hdfs dfs -mkdir /user/data/experimental/analytics | ||
| hdfs dfs -mkdir /user/data/experimental/reporting | ||
| hdfs dfs -mkdir /user/data/experimental/sandbox | ||
| hdfs dfs -mkdir /user/data/experimental/dqerror | ||
| hdfs dfs -mkdir /user/data/experimental/mdm_oldvalue | ||
| hdfs dfs -mkdir /user/data/experimental/backup | ||
| echo "Creating HDFS Paths: FINISH" | ||
| echo "STARTING HIVE SETUP" | ||
| hive -e "CREATE DATABASE production_master" | ||
| hive -e "CREATE DATABASE experimental_master" | ||
| hive -e "CREATE DATABASE production_dim" | ||
| hive -e "CREATE DATABASE experimental_dim" | ||
| hive -e "CREATE DATABASE production_analytics" | ||
| hive -e "CREATE DATABASE experimental_analytics" | ||
| hive -e "CREATE DATABASE production_reporting" | ||
| hive -e "CREATE DATABASE experimental_reporting" | ||
| hive -e "CREATE DATABASE production_sandbox" | ||
| hive -e "CREATE DATABASE experimental_sandbox" | ||
| hive -e "CREATE DATABASE production_DQError" | ||
| hive -e "CREATE DATABASE experimental_DQError" | ||
| hive -e "CREATE DATABASE production_mdm_oldvalue" | ||
| hive -e "CREATE DATABASE experimental_mdm_oldvalue" | ||
| echo "STARTING HIVE SETUP" |
14 changes: 14 additions & 0 deletions
14
src/main/resources/Instalacion/upgrade from 2.4 to 2.5/huemul_bdg_upgrade_2.4_to_2.5.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
|
|
||
| ALTER TABLE control_columns add column_datatypedeploy varchar(50); | ||
|
|
||
| UPDATE control_columns SET column_datatypedeploy = column_datatype WHERE column_datatypedeploy IS NULL; | ||
|
|
||
|
|
||
| UPDATE control_config | ||
| SET version_mayor = 2 | ||
| ,version_minor = 5 | ||
| ,version_patch = 0 | ||
| where config_id = 1; | ||
|
|
||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
src/main/scala/com/huemulsolutions/bigdata/common/huemul_ExternalHWC.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package com.huemulsolutions.bigdata.common | ||
|
|
||
| import com.hortonworks.hwc.HiveWarehouseSession | ||
| import org.apache.spark.sql._ | ||
| import com.hortonworks.spark.sql.hive.llap.HiveWarehouseSessionImpl | ||
|
|
||
| /** | ||
| * connect using Hortonworks Warehouse connector | ||
| * used by huemul_ExternalDB.Using_HWC | ||
| */ | ||
| class huemul_ExternalHWC(huemulBigDataGov: huemul_BigDataGovernance) extends Serializable { | ||
| @transient private var _HWC_Hive: HiveWarehouseSessionImpl = null | ||
| def getHWC_Hive: HiveWarehouseSessionImpl = { | ||
| if (_HWC_Hive != null) | ||
| return _HWC_Hive | ||
|
|
||
| _HWC_Hive = HiveWarehouseSession.session(huemulBigDataGov.spark).build() | ||
|
|
||
|
|
||
| return _HWC_Hive | ||
| } | ||
|
|
||
| def execute_NoResulSet(sql: String): Boolean = { | ||
| val _hive = getHWC_Hive | ||
| if (_hive == null) | ||
| sys.error("can't connect with HIVE, HiveWarehouseSession.session doesnt works") | ||
|
|
||
| return _hive.executeUpdate(sql) | ||
| } | ||
|
|
||
| def close { | ||
| val _hive = getHWC_Hive | ||
| if (_hive != null) | ||
| _hive.session().close() | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.