notebooks/example_hfhub.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "08adc1a8-2c82-4d3f-a353-52cde177b227",
   "metadata": {},
   "source": [
    "# Example on how load and save from Hugging Face Hub\n",
    "\n",
    "[Source](https://colab.research.google.com/drive/1Uqy8i6G4og6awtNmJGatqek2mgBzvvXu?usp=sharing#scrollTo=F7ouGeDwYzZt)\n",
    "\n",
    "Author: [Pavel Iakubovskii](https://www.linkedin.com/in/pavel-iakubovskii/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d889cdf0-c38f-49f0-9273-91f92e953997",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/Caskroom/miniconda/base/envs/albumentations_examples/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from huggingface_hub import notebook_login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd174f54-b775-4cb4-88e4-343ed7586131",
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "The `notebook_login` function can only be used in a notebook (Jupyter or Colab) and you need the `ipywidgets` module: `pip install ipywidgets`.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/albumentations_examples/lib/python3.9/site-packages/huggingface_hub/_login.py:340\u001b[0m, in \u001b[0;36mnotebook_login\u001b[0;34m(new_session, write_permission)\u001b[0m\n\u001b[1;32m    339\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 340\u001b[0m     \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mipywidgets\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mwidgets\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mas\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mwidgets\u001b[39;00m  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[1;32m    341\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mIPython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdisplay\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m display  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'ipywidgets'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[4], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mnotebook_login\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/albumentations_examples/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py:101\u001b[0m, in \u001b[0;36m_deprecate_arguments.<locals>._inner_deprecate_positional_args.<locals>.inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     99\u001b[0m         message \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m+\u001b[39m custom_message\n\u001b[1;32m    100\u001b[0m     warnings\u001b[38;5;241m.\u001b[39mwarn(message, \u001b[38;5;167;01mFutureWarning\u001b[39;00m)\n\u001b[0;32m--> 101\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/albumentations_examples/lib/python3.9/site-packages/huggingface_hub/utils/_deprecation.py:31\u001b[0m, in \u001b[0;36m_deprecate_positional_args.<locals>._inner_deprecate_positional_args.<locals>.inner_f\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     29\u001b[0m extra_args \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlen\u001b[39m(args) \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mlen\u001b[39m(all_args)\n\u001b[1;32m     30\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m extra_args \u001b[38;5;241m<\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[0;32m---> 31\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mf\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     32\u001b[0m \u001b[38;5;66;03m# extra_args > 0\u001b[39;00m\n\u001b[1;32m     33\u001b[0m args_msg \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m     34\u001b[0m     \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;132;01m{\u001b[39;00marg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(arg, \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00marg\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     35\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m name, arg \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(kwonly_args[:extra_args], args[\u001b[38;5;241m-\u001b[39mextra_args:])\n\u001b[1;32m     36\u001b[0m ]\n",
      "File \u001b[0;32m/opt/homebrew/Caskroom/miniconda/base/envs/albumentations_examples/lib/python3.9/site-packages/huggingface_hub/_login.py:343\u001b[0m, in \u001b[0;36mnotebook_login\u001b[0;34m(new_session, write_permission)\u001b[0m\n\u001b[1;32m    341\u001b[0m     \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21;01mIPython\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mdisplay\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m display  \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[1;32m    342\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m:\n\u001b[0;32m--> 343\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[1;32m    344\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mThe `notebook_login` function can only be used in a notebook (Jupyter or\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    345\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m Colab) and you need the `ipywidgets` module: `pip install ipywidgets`.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    346\u001b[0m     )\n\u001b[1;32m    347\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m new_session \u001b[38;5;129;01mand\u001b[39;00m get_token() \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    348\u001b[0m     logger\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mUser is already logged in.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[0;31mImportError\u001b[0m: The `notebook_login` function can only be used in a notebook (Jupyter or Colab) and you need the `ipywidgets` module: `pip install ipywidgets`."
     ]
    }
   ],
   "source": [
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe64b254-dc4a-423c-832c-271621b46e89",
   "metadata": {},
   "outputs": [],
   "source": [
    "import albumentations as A\n",
    "\n",
    "transform = A.Compose(\n",
    "    [\n",
    "        A.RandomCrop(256, 256),\n",
    "        A.HorizontalFlip(),\n",
    "        A.RandomBrightnessContrast(),\n",
    "        A.RGBShift(),\n",
    "        A.Normalize(),\n",
    "    ],\n",
    "    seed=137,\n",
    "    strict=True,\n",
    ")\n",
    "\n",
    "evaluation_transform = A.Compose(\n",
    "    [\n",
    "        A.PadIfNeeded(256, 256),\n",
    "        A.Normalize(),\n",
    "    ],\n",
    "    seed=137,\n",
    "    strict=True,\n",
    ")\n",
    "\n",
    "transform.save_pretrained(\"qubvel-hf/albu\", key=\"train\")\n",
    "# ^ this will save the transform to a directory \"qubvel-hf/albu\" with filename \"albumentations_config_train.json\"\n",
    "\n",
    "transform.save_pretrained(\"qubvel-hf/albu\", key=\"train\", push_to_hub=True)\n",
    "# ^ this will save the transform to a directory \"qubvel-hf/albu\" with filename \"albumentations_config_train.json\"\n",
    "# + push the transform to the Hub to the repository \"qubvel-hf/albu\"\n",
    "\n",
    "transform.push_to_hub(\"qubvel-hf/albu\", key=\"train\")\n",
    "# ^ this will push the transform to the Hub to the repository \"qubvel-hf/albu\" (without saving it locally)\n",
    "\n",
    "loaded_transform = A.Compose.from_pretrained(\"qubvel-hf/albu\", key=\"train\")\n",
    "# ^ this will load the transform from local folder if exist or from the Hub repository \"qubvel-hf/albu\"\n",
    "\n",
    "evaluation_transform.save_pretrained(\"qubvel-hf/albu\", key=\"eval\", push_to_hub=True)\n",
    "# ^ this will save the transform to a directory \"qubvel-hf/albu\" with filename \"albumentations_config_eval.json\"\n",
    "\n",
    "loaded_evaluation_transform = A.Compose.from_pretrained(\"qubvel-hf/albu\", key=\"eval\")\n",
    "# ^ this will load the transform from the Hub repository \"qubvel-hf/albu\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1a32b401-8e5e-43be-ac54-1709274ceedd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# check\n",
    "import numpy as np\n",
    "\n",
    "image = np.random.randint(0, 255, (100, 200, 3), dtype=np.uint8)\n",
    "\n",
    "preprocessed_image_1 = evaluation_transform(image=image)[\"image\"]\n",
    "preprocessed_image_2 = loaded_evaluation_transform(image=image)[\"image\"]\n",
    "\n",
    "assert np.allclose(preprocessed_image_1, preprocessed_image_2)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}