01_standalone_examples/azure-databricks/Azure Databricks Tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "7aa2863e-624c-495f-b15a-27daf382d4b8",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "#setting up lakeFS end point access and secret in order to later configure the python client\n",
    "lakefsEndPoint = 'https://YourEndPoint/' # e.g. 'https://username.azure_region_name.lakefscloud.io'\n",
    "lakefsAccessKey = 'AKIAlakeFSAccessKey'\n",
    "lakefsSecretKey = 'lakeFSSecretKey'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "bcd6f8b3-c96e-4b1b-bf59-b89b4091de1e",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Configuring Python Client\n",
    "import lakefs\n",
    "from lakefs.client import Client\n",
    "\n",
    "clt = Client(\n",
    "   host=lakefsEndPoint,\n",
    "   username=lakefsAccessKey,\n",
    "   password=lakefsSecretKey,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "638f2aa1-0531-4a05-b345-0aba97be6859",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Configuring variables\n",
    "repositoryName = \"learn-lakefs-repo01\" # the name of the lakeFS repository\n",
    "storageNamespace = \"https://storage-account-name.blob.core.windows.net/storage-container-name/\"  + repositoryName # Unique per repository\n",
    "sourceBranch = \"main\"\n",
    "dataPath = \"product-reviews\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "5850880d-220a-44fe-8155-c66c633d2571",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Creating a repository \n",
    "repo = lakefs.Repository(\n",
    "  repositoryName,\n",
    "  client=clt).create(\n",
    "    storage_namespace=storageNamespace,\n",
    "    default_branch=sourceBranch,\n",
    "    exist_ok=True)\n",
    "branchMain = repo.branch(sourceBranch)\n",
    "print(repo)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "25fc4b52-5c18-4cde-ad6b-fe52e5eb3d1e",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Reading data from databricks datasets and inserging into the newly created repository (Creating initial data)\n",
    "import_data_path = \"/databricks-datasets/amazon/test4K/\"\n",
    "df = spark.read.parquet(import_data_path)\n",
    "df.write.format(\"parquet\").save(\"lakefs://{}/{}/{}\".format(repositoryName,sourceBranch,dataPath))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "86a5cb71-561f-4ae3-bef0-2d76f4c1ba83",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Commiting the changes\n",
    "branchMain.commit(message='Uploading intial data into lakefs', \n",
    "        metadata={'using': 'python_api'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {
      "byteLimit": 2048000,
      "rowLimit": 10000
     },
     "inputWidgets": {},
     "nuid": "bb90ee4b-2a58-44c3-8111-0fc709ef273c",
     "showTitle": false,
     "title": ""
    }
   },
   "outputs": [],
   "source": [
    "# Review production Data from your production \"main\" branch\n",
    "\n",
    "# Note - This example uses static strings instead of parameters for an easier read\n",
    "\n",
    "df = spark.read.parquet(\"lakefs://learn-lakefs-repo01/main/product-reviews/\")\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "application/vnd.databricks.v1+cell": {
     "cellMetadata": {},
     "inputWidgets": {},
     "nuid": "1fbbdd5c-69cd-49d8-9a39-afe7493921be",
     "showTitle": false,
     "title": ""
    }
   },
   "source": [
    "#### More Questions?\n",
    "\n",
    "###### Join the [lakeFS Slack group](https://lakefs.io/slack)"
   ]
  }
 ],
 "metadata": {
  "application/vnd.databricks.v1+notebook": {
   "dashboards": [],
   "language": "python",
   "notebookMetadata": {
    "mostRecentlyExecutedCommandWithImplicitDF": {
     "commandId": 2623252509645554,
     "dataframes": [
      "_sqldf"
     ]
    },
    "pythonIndentUnit": 2
   },
   "notebookName": "Azure Databricks Tutorial",
   "widgets": {}
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}