savi-create-data-sources

#!/bin/bash -f
#
# This script creates the data source objects that store the location of the
# specified input and output data.
#
# Prerequisite:
# * Your Testbed's credentials was exported.

desc="This script creates the data source objects that store the location of the
specified input and output data.
"

script_name=`basename $0`
usage="Usage:
1. ${script_name} swift <container> <input_object> <output_object>
2. ${script_name} hdfs <cluster_name> <input_path> <output_path>
3. ${script_name} help
"

if [[ -n "$1" && $1 == "help" ]]; then
    echo "${desc}"
    echo "${usage}"
    exit
fi

source /home/savitb/bin/functions

function command_output_desc() {
    OUT=$?
    if [ $OUT -eq 0 ];then
       echo "Succeeded."
    else
       echo $1
       blue_desc "Failed!"
       exit 1
    fi
}

# Check the prerequisite.
if [ -z "$OS_USERNAME" ]; then
    echo "The environment variable OS_USERNAME is not set."
    exit 1
fi

if [ -z "$OS_PASSWORD" ]; then
    echo "The environment variable OS_PASSWORD is not set."
    exit 1
fi

if [ -z "$OS_AUTH_URL" ]; then
    echo "The environment variable OS_AUTH_URL is not set."
    exit 1
fi

if [ -z "$OS_TENANT_NAME" ]; then
    echo "The environment variable OS_TENANT_NAME is not set."
    exit 1
fi

if [ -z "$OS_REGION_NAME" ]; then
    echo "The environment variable OS_REGION_NAME is not set."
    exit 1
fi

# Check parameters.
unset ERR
if [ -n "$1" ]; then
    ds_type=$1
    shift
else
    ERR=1
fi

if [ -n "$1" ]; then
    second_param=$1
    shift
else
    ERR=1
fi

if [ -n "$1" ]; then
    input_path=$1
    shift
else
    ERR=1
fi

if [ -n "$1" ]; then
    output_path=$1
    shift
else
    ERR=1
fi

if [ -n "$ERR" ]; then
    echo "One or multiple arguments were missing."
    echo ""
    echo "${usage}"
    exit 1
fi

# Remove the leading character '/' at path if any.
# For example, convert '/path/to/file' to 'path/to/file'.
input_path=`echo ${input_path} | sed 's/^\///'`
output_path=`echo ${output_path} | sed 's/^\///'`

input_file=`basename ${input_path}`
output_file=`basename ${output_path}`

blue_desc_title "  Creating data sources"

if [ $ds_type = "swift" ]; then
    container_name="${second_param}"
    input_url="swift://${container_name}.sahara/${input_path}"
    output_url="swift://${container_name}.sahara/${output_path}"

    green_desc_title "1. Creating the input data source object ..."

    # Create the data source object that store the location of the input data.
    command="sahara data-source-create --name ${input_file} --type ${ds_type} --user ${OS_USERNAME} --password ${OS_PASSWORD}
             --url ${input_url}"
    yellow_desc "${command}"
    output=`eval ${command}`
    command_output_desc "${output}"

    green_desc_title "2. Creating the output data source object ..."

    # Create the data source object that store the location of the output data.
    command="sahara data-source-create --name ${output_file} --type ${ds_type} --user ${OS_USERNAME} --password ${OS_PASSWORD}
             --url ${output_url}"
    yellow_desc "${command}"
    output=`eval ${command}`
    command_output_desc "${output}"
elif [ $ds_type = "hdfs" ]; then
    cluster_name="${second_param}"
    # Retrieve the IP address of the master node of the cluster.
    master_ip=`sahara cluster-show --name ${cluster_name} 2>/dev/null | awk -F'[:/]' '/Oozie/{print $5}'`
    if [ -z "$master_ip" ]; then
        echo "Error: cannot find cluster '${cluster_name}'."
        exit 1
    fi
    input_url="hdfs://${master_ip}:9000/${input_path}"
    output_url="hdfs://${master_ip}:9000/${output_path}"

    green_desc_title "1. Creating the input data source object ..."

    # Create the data source object that store the location of the input data.
    command="sahara data-source-create --name ${input_file} --type ${ds_type} --url ${input_url}"
    yellow_desc "${command}"
    output=`eval ${command}`
    command_output_desc "${output}"

    green_desc_title "2. Creating the output data source object ..."

    # Create the data source object that store the location of the output data.
    command="sahara data-source-create --name ${output_file} --type ${ds_type} --url ${output_url}"
    yellow_desc "${command}"
    output=`eval ${command}`
    command_output_desc "${output}"
else
    echo "Invalid data source type '${ds_type}'."
    echo ""
    echo "${usage}"
    exit 1
fi

echo ""
blue_desc "Data sources were created."