RANGER-2497 : Support Azure Key Vault for storing master keys of Rang…

…er KMS Signed-off-by: Mehul Parikh <[email protected]>
atlanhq · Sep 29, 2019 · 02640d3 · 02640d3
1 parent b69227e
commit 02640d3
Show file tree

Hide file tree

Showing 16 changed files with 2,547 additions and 859 deletions.
diff --git a/LICENSE.txt b/LICENSE.txt
@@ -217,7 +217,8 @@ The Apache Ranger project bundles the following files under the MIT License:
 This product includes jQuery (http://jquery.org ), Copyright © 2014, John Resig.
 This product includes jQuery UI (http://jqueryui.com ), Copyright © 2013 jQuery Foundation.  
 This product includes Backbone (http://backbonejs.org ), Copyright © 2010-2014 Jeremy Ashkenas, DocumentCloud.
-This product includes underscore (http:underscorejs.org ), Copyright © 2009-2014 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors.  
+This product includes underscore (http:underscorejs.org ), Copyright © 2009-2014 Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors.
+This product includes Azure/azure-sdk-for-java (https://github.com/Azure/azure-sdk-for-java/blob/master/LICENSE.txt ), Copyright (c) 2015 Microsoft.  
 This product includes Backbone.Marionette (http://marionettejs.com/ ) Copyright (c)2013 Derick Bailey, Muted Solutions, LLC.  
 This product includes Backbone.Wreqr (http://marionettejs.com/ ) Copyright ©,2012 Derick Bailey, Muted Solutions, LLC.
 This product includes Backbone.BabySitter (http://marionettejs.com/ ), Copyright ©2013 Derick Bailey, Muted Solutions, LLC.  

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -3,3 +3,4 @@ Copyright 2014-2019 The Apache Software Foundation
 
 This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
 This product includes software developed by Spring Security Project (http://www.springframework.org/security)
+This product includes Azure/azure-sdk-for-java (https://github.com/Azure/azure-sdk-for-java/blob/master/LICENSE.txt ), Copyright (c) 2015 Microsoft.
diff --git a/kms/config/kms-webapp/dbks-site.xml b/kms/config/kms-webapp/dbks-site.xml
@@ -231,6 +231,63 @@
   </property>
 
   <!-- Key-Secure Config END-->
+   <!--Azure Key Vault START-->
+   <property>
+        <name>ranger.kms.azurekeyvault.enabled</name>
+        <value>false</value>
+        <description>Flag for Azure Key Vault</description>
+  </property>
+  <property>
+        <name>ranger.kms.azure.keyvault.ssl.enabled</name>
+        <value>false</value>
+        <description>Flag for Azure authentication via certificate or password</description>
+  </property>
+  <property>
+        <name>ranger.kms.azure.client.id</name>
+        <value></value>
+        <description>Azure Client Id</description>
+  </property>
+  <property>
+        <name>ranger.kms.azure.client.secret</name>
+        <value></value>
+        <description>Azure Client Secret</description>
+  </property>
+    <property>
+        <name>ranger.kms.azure.client.secret.alias</name>
+        <value>ranger.ks.azure.client.secret</value>
+        <description>Azure Client Secret Alias</description>
+  </property>
+  <property>
+        <name>ranger.kms.azure.keyvault.certificate.path</name>
+        <value>/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx</value>
+        <description>Azure key vault cerificate path</description>
+  </property>
+  <property>
+        <name>ranger.kms.azure.keyvault.certificate.password</name>
+        <value></value>
+        <description>Azure key vault cerificate password</description>
+  </property>
+   <property>
+        <name>ranger.kms.azure.masterkey.name</name>
+        <value></value>
+        <description>Azure master key name</description>
+  </property>
+  <property>
+        <name>ranger.kms.azure.masterkey.type</name>
+        <value></value>
+        <description>Azure key type: RSA, RSA_HSM, EC, EC_HSM</description>
+  </property>
+    <property>
+        <name>ranger.kms.azure.zonekey.encryption.algorithm</name>
+        <value></value>
+        <description>Encryption Algo : RSA_OAEP, RSA_OAEP_256, RSA1_5</description>
+  </property>
+   <property>
+        <name>ranger.kms.azurekeyvault.url</name>
+        <value></value>
+        <description>Azure Key Vault url</description>
+  </property>
+   <!--Azure Key Vault END-->
 
   <!-- HSM Config -->
   <property>

diff --git a/kms/pom.xml b/kms/pom.xml
@@ -271,6 +271,56 @@
             <version>${derby.version}</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+		    <groupId>com.microsoft.azure</groupId>
+		    <artifactId>azure</artifactId>
+		    <version>${com.microsoft.azure.version}</version>
+		</dependency>
+		<dependency>
+			<groupId>com.microsoft.azure</groupId>
+			<artifactId>azure-keyvault</artifactId>
+			<version>${com.microsoft.azure.azure-keyvault.version}</version>
+		</dependency>
+		<dependency>
+		    <groupId>com.microsoft.azure</groupId>
+		    <artifactId>azure-mgmt-keyvault</artifactId>
+		    <version>${com.microsoft.azure.azure-mgmt-keyvault.version}</version>
+		</dependency>
+		<dependency>
+		    <groupId>com.microsoft.rest</groupId>
+		    <artifactId>client-runtime</artifactId>
+		    <version>${com.microsoft.rest.client-runtime.version}</version>
+		</dependency>
+		<dependency>
+		    <groupId>com.microsoft.azure</groupId>
+		    <artifactId>azure-client-runtime</artifactId>
+		    <version>${com.microsoft.azure.azure-client-runtime.version}</version>
+		</dependency>
+		<dependency>
+		    <groupId>com.microsoft.azure</groupId>
+   			<artifactId>adal4j</artifactId>
+    		<version>${com.microsoft.azure.adal4j.version}</version>
+		</dependency>
+		<dependency>
+		    <groupId>io.reactivex</groupId>
+		    <artifactId>rxjava</artifactId>
+		    <version>${io.reactivex.rxjava.version}</version>
+		</dependency>
+		<dependency>
+		    <groupId>net.minidev</groupId>
+		    <artifactId>asm</artifactId>
+		    <version>${net.minidev.asm.version}</version>
+		</dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcprov-jdk15on</artifactId>
+            <version>${org.bouncycastle.bcprov-jdk15on}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.bouncycastle</groupId>
+            <artifactId>bcpkix-jdk15on</artifactId>
+            <version>${org.bouncycastle.bcpkix-jdk15on}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.directory.api</groupId>
             <artifactId>api-i18n</artifactId>

diff --git a/kms/scripts/DBMKTOAZUREKEYVAULT.sh b/kms/scripts/DBMKTOAZUREKEYVAULT.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -------------------------------------------------------------------------------------
+if [ "$JAVA_HOME" != "" ]; then
+    export PATH=$JAVA_HOME/bin:$PATH
+else
+    exit ;
+fi
+
+RANGER_KMS_HOME=`dirname $0`
+cp="${RANGER_KMS_HOME}/cred/lib/*:${RANGER_KMS_HOME}/./ews/webapp/WEB-INF/classes/conf/:${RANGER_KMS_HOME}/ews/webapp/config:${RANGER_KMS_HOME}/ews/lib/*:${RANGER_KMS_HOME}/ews/webapp/lib/*:${RANGER_KMS_HOME}/ews/webapp/META-INF"
+java -cp "${cp}" org.apache.hadoop.crypto.key.DBToAzureKeyVault ${1} ${2} ${3} ${4} ${5} ${6} ${7} ${8} ${9}
diff --git a/kms/scripts/install.properties b/kms/scripts/install.properties
@@ -99,6 +99,22 @@ KEYSECURE_HOSTNAME=SunPKCS11-keysecurehn
 KEYSECURE_MASTER_KEY_SIZE=256
 KEYSECURE_LIB_CONFIG_PATH=/opt/safenetConf/64/8.3.1/sunpkcs11.cfg
 
+#------------------------- Ranger Azure Key Vault ------------------------------
+AZURE_KEYVAULT_ENABLED=false
+AZURE_KEYVAULT_SSL_ENABLED=false
+AZURE_CLIENT_ID=50fd7ca6-fd4f-4785-a13f-1a6cc4e95e42
+AZURE_CLIENT_SECRET=<AzureKeyVaultPassword>
+AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=/home/machine/Desktop/azureAuthCertificate/keyvault-MyCert.pfx
+# Initialize below prop if your certificate file has any password
+#AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=certPass
+AZURE_MASTERKEY_NAME=RangerMasterKey
+# E.G. RSA, RSA_HSM, EC, EC_HSM, OCT
+AZURE_MASTER_KEY_TYPE=RSA
+# E.G. RSA_OAEP, RSA_OAEP_256, RSA1_5, RSA_OAEP 
+ZONE_KEY_ENCRYPTION_ALGO=RSA_OAEP
+AZURE_KEYVAULT_URL=https://shahkeyvault.vault.azure.net/
+
+
 #
 # ------- UNIX User CONFIG ----------------
 #

diff --git a/kms/scripts/setup.sh b/kms/scripts/setup.sh
@@ -100,6 +100,17 @@ KEYSECURE_HOSTNAME=$(get_prop 'KEYSECURE_HOSTNAME' $PROPFILE)
 KEYSECURE_MASTER_KEY_SIZE=$(get_prop 'KEYSECURE_MASTER_KEY_SIZE' $PROPFILE)
 KEYSECURE_LIB_CONFIG_PATH=$(get_prop 'KEYSECURE_LIB_CONFIG_PATH' $PROPFILE)
 
+AZURE_KEYVAULT_ENABLED=$(get_prop 'AZURE_KEYVAULT_ENABLED' $PROPFILE)
+AZURE_KEYVAULT_SSL_ENABLED=$(get_prop 'AZURE_KEYVAULT_SSL_ENABLED' $PROPFILE)
+AZURE_CLIENT_ID=$(get_prop 'AZURE_CLIENT_ID' $PROPFILE)
+AZURE_CLIENT_SECRET=$(get_prop 'AZURE_CLIENT_SECRET' $PROPFILE)
+AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH=$(get_prop 'AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH' $PROPFILE)
+AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD=$(get_prop 'AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD' $PROPFILE)
+AZURE_MASTERKEY_NAME=$(get_prop 'AZURE_MASTERKEY_NAME' $PROPFILE)
+AZURE_MASTER_KEY_TYPE=$(get_prop 'AZURE_MASTER_KEY_TYPE' $PROPFILE)
+ZONE_KEY_ENCRYPTION_ALGO=$(get_prop 'ZONE_KEY_ENCRYPTION_ALGO' $PROPFILE)
+AZURE_KEYVAULT_URL=$(get_prop 'AZURE_KEYVAULT_URL' $PROPFILE)
+
 kms_principal=$(get_prop 'kms_principal' $PROPFILE)
 kms_keytab=$(get_prop 'kms_keytab' $PROPFILE)
 hadoop_conf=$(get_prop 'hadoop_conf' $PROPFILE)
@@ -224,6 +235,16 @@ password_validation_safenet_keysecure(){
         fi
 }
 
+azure_client_secret_validation(){
+        if [ -z "$1" ]
+        then
+                log "[I] Blank password is not allowed for" $2". Please enter valid password."
+                exit 1
+        else
+                log "[I]" $2 "password validated."
+        fi
+}
+
 init_variables(){
 	curDt=`date '+%Y%m%d%H%M%S'`
 
@@ -571,9 +592,13 @@ update_properties() {
         KEYSECURE_PASSWD="ranger.kms.keysecure.login.password"
         KEYSECURE_PASSWORD_ALIAS="ranger.ks.login.password"
 
+	AZURE_CLIENT_SEC="ranger.kms.azure.client.secret"
+	AZURE_CLIENT_SECRET_ALIAS="ranger.ks.azure.client.secret"
+
 
         HSM_ENABLED=`echo $HSM_ENABLED | tr '[:lower:]' '[:upper:]'`
         KEYSECURE_ENABLED=`echo $KEYSECURE_ENABLED | tr '[:lower:]' '[:upper:]'`
+	AZURE_KEYVAULT_ENABLED=`echo $AZURE_KEYVAULT_ENABLED | tr '[:lower:]' '[:upper:]'`
 
 	if [ "${keystore}" != "" ]
 	then
@@ -613,6 +638,20 @@ update_properties() {
                         updatePropertyToFilePy $propertyName $newPropertyValue $to_file
                 fi
 
+		if [ "${AZURE_KEYVAULT_ENABLED}" == "TRUE" ]
+                then
+                        azure_client_secret_validation "$AZURE_CLIENT_SECRET" "Azure Client Secret"
+                        $PYTHON_COMMAND_INVOKER ranger_credential_helper.py -l "cred/lib/*" -f "$keystore" -k "${AZURE_CLIENT_SECRET_ALIAS}" -v "${AZURE_CLIENT_SECRET}" -c 1
+
+                        propertyName=ranger.kms.azure.client.secret.alias
+                        newPropertyValue="${AZURE_CLIENT_SECRET_ALIAS}"
+                        updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+                        propertyName=ranger.kms.azure.client.secret
+                        newPropertyValue="_"
+                        updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+                fi
+
 
 		propertyName=ranger.ks.jpa.jdbc.credential.alias
 		newPropertyValue="${DB_CREDENTIAL_ALIAS}"
@@ -650,6 +689,10 @@ update_properties() {
                 newPropertyValue="${KEYSECURE_PASSWORD}"
                 updatePropertyToFilePy $propertyName $newPropertyValue $to_file
 
+		propertyName="${AZURE_CLIENT_SEC}"
+                newPropertyValue="${AZURE_CLIENT_SECRET}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
 	fi
 
 	if test -f $keystore; then
@@ -753,6 +796,55 @@ update_properties() {
 
         fi
 
+	########### AZURE KEY VAULT #################
+
+
+        if [ "${AZURE_KEYVAULT_ENABLED}" != "TRUE" ]
+        then
+                propertyName=ranger.kms.azurekeyvault.enabled
+                newPropertyValue="false"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+        else
+                propertyName=ranger.kms.azurekeyvault.enabled
+                newPropertyValue="true"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+		propertyName=ranger.kms.azure.keyvault.ssl.enabled
+                newPropertyValue="${AZURE_KEYVAULT_SSL_ENABLED}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+                propertyName=ranger.kms.azure.client.id
+                newPropertyValue="${AZURE_CLIENT_ID}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+		propertyName=ranger.kms.azure.keyvault.certificate.path
+                newPropertyValue="${AZURE_AUTH_KEYVAULT_CERTIFICATE_PATH}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+		propertyName=ranger.kms.azure.keyvault.certificate.password
+                newPropertyValue="${AZURE_AUTH_KEYVAULT_CERTIFICATE_PASSWORD}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+
+                propertyName=ranger.kms.azure.masterkey.name
+                newPropertyValue="${AZURE_MASTERKEY_NAME}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+		propertyName=ranger.kms.azure.masterkey.type
+                newPropertyValue="${AZURE_MASTER_KEY_TYPE}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+                propertyName=ranger.kms.azure.zonekey.encryption.algorithm
+                newPropertyValue="${ZONE_KEY_ENCRYPTION_ALGO}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+                propertyName=ranger.kms.azurekeyvault.url
+                newPropertyValue="${AZURE_KEYVAULT_URL}"
+                updatePropertyToFilePy $propertyName $newPropertyValue $to_file
+
+        fi
+
+
 	to_file_kms_site=$PWD/ews/webapp/WEB-INF/classes/conf/ranger-kms-site.xml
     if test -f $to_file_kms_site; then
 		log "[I] $to_file_kms_site file found"
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3,3 +3,4 @@ Copyright 2014-2019 The Apache Software Foundation

		This product includes software developed at The Apache Software Foundation (http://www.apache.org/).
		This product includes software developed by Spring Security Project (http://www.springframework.org/security)
		This product includes Azure/azure-sdk-for-java (https://github.com/Azure/azure-sdk-for-java/blob/master/LICENSE.txt ), Copyright (c) 2015 Microsoft.