From 6e731e9d3c2f8810785aafb4213fdc6444fc31fa Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 3 Jun 2024 09:19:12 +0200 Subject: [PATCH 1/8] Move transformers example --- .../Comet_with_Hugging_Face_Trainer.ipynb | 351 ------- .../Comet_with_Hugging_Face_Trainer.ipynb | 902 ++++++++++++++++++ .../README.md | 0 .../data/title_conference.csv | 0 .../requirements.txt | 2 +- .../transformers-distilbert-fine-tuning.py | 0 6 files changed, 903 insertions(+), 352 deletions(-) delete mode 100644 integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb create mode 100644 integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb rename integrations/model-training/{hugging_face => transformers}/transformers-distilbert-fine-tuning/README.md (100%) rename integrations/model-training/{hugging_face => transformers}/transformers-distilbert-fine-tuning/data/title_conference.csv (100%) rename integrations/model-training/{hugging_face => transformers}/transformers-distilbert-fine-tuning/requirements.txt (69%) rename integrations/model-training/{hugging_face => transformers}/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py (100%) diff --git a/integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb b/integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb deleted file mode 100644 index d958ee0d..00000000 --- a/integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb +++ /dev/null @@ -1,351 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "UG0Hxf-nFKG2" - }, - "source": [ - "" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "5L-2VqFDWgGx" - }, - "source": [ - "[Hugging Face](https://huggingface.co/docs) is a community and data science platform that provides tools that enable users to build, train and deploy ML models based on open source (OS) code and technologies. Primarily known for their `transformers` library, Hugging Face has helped democratized access to these models by providing a unified API to train and evaluate a number of popular models for NLP.\n", - "\n", - "Comet integrates with Hugging Face's `Trainer` object, allowing you to log your model parameters, metrics, and assets such as model checkpoints. Learn more about our integration [here](https://www.comet.com/docs/v2/integrations/ml-frameworks/huggingface/)\n", - "\n", - "Curious about how Comet can help you build better models, faster? Find out more about [Comet](https://www.comet.com/site/products/ml-experiment-tracking/?utm_campaign=transformers&utm_medium=colab) and our [other integrations](https://www.comet.ml/docs/v2/integrations/overview/)\n", - "\n", - "\n", - "Get a preview for what's to come. Check out a completed experiment created from this notebook [here](https://www.comet.com/examples/comet-examples-transformers-trainer/3992ddee441f446bbb65c3cc4c8bd33b)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KUdcwLCWIz2n" - }, - "source": [ - "# Install Comet and Dependencies" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "nGDxYlCEKHMt" - }, - "outputs": [], - "source": [ - "%pip install comet_ml torch datasets transformers scikit-learn accelerate" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "3VaHie31I3r_" - }, - "source": [ - "# Initialize Comet" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "8twVwYTIKK_x" - }, - "outputs": [], - "source": [ - "import comet_ml\n", - "\n", - "comet_ml.init(project_name=\"comet-examples-transfomers-trainer\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "y-isTv4sTU1n" - }, - "source": [ - "# Set Model Type" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "2R9ckDVbTWaN" - }, - "outputs": [], - "source": [ - "PRE_TRAINED_MODEL_NAME = \"distilbert/distilroberta-base\"\n", - "SEED = 42" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "dJRfuqCXI948" - }, - "source": [ - "# Load Data" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "AD_Ejs6sKTRl" - }, - "outputs": [], - "source": [ - "from transformers import AutoTokenizer, Trainer, TrainingArguments\n", - "from datasets import load_dataset\n", - "\n", - "raw_datasets = load_dataset(\"imdb\")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "IDXLkGRnI_jn" - }, - "source": [ - "# Setup Tokenizer" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "DI24QdbPKoO9" - }, - "outputs": [], - "source": [ - "tokenizer = AutoTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "flBnDDk4M8Yo" - }, - "outputs": [], - "source": [ - "def tokenize_function(examples):\n", - " return tokenizer(\n", - " examples[\"text\"],\n", - " padding=\"max_length\",\n", - " truncation=True,\n", - " )\n", - "\n", - "\n", - "tokenized_datasets = raw_datasets.map(tokenize_function, batched=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "vhN2owT-tOcv" - }, - "outputs": [], - "source": [ - "from transformers import DataCollatorWithPadding\n", - "\n", - "data_collator = DataCollatorWithPadding(tokenizer=tokenizer)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jMX-ysNYOCN4" - }, - "source": [ - "# Create Sample Datasets" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yBYSBPoZOEFq" - }, - "source": [ - "For this guide, we are only going to sample 200 examples from our dataset. " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "urC6KnPbNET3" - }, - "outputs": [], - "source": [ - "train_dataset = tokenized_datasets[\"train\"].shuffle(seed=SEED).select(range(200))\n", - "eval_dataset = tokenized_datasets[\"test\"].shuffle(seed=SEED).select(range(200))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "HU4ggdjCJJT1" - }, - "source": [ - "# Setup Transformer Model" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sythIXibNmFD" - }, - "outputs": [], - "source": [ - "from transformers import AutoModelForSequenceClassification\n", - "\n", - "model = AutoModelForSequenceClassification.from_pretrained(\n", - " PRE_TRAINED_MODEL_NAME, num_labels=2\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EXUjopsZpo-D" - }, - "source": [ - "# Setup Evaluation Function" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Gl6m6IJUporu" - }, - "outputs": [], - "source": [ - "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n", - "\n", - "\n", - "def get_example(index):\n", - " return eval_dataset[index][\"text\"]\n", - "\n", - "\n", - "def compute_metrics(pred):\n", - " experiment = comet_ml.get_global_experiment()\n", - "\n", - " labels = pred.label_ids\n", - " preds = pred.predictions.argmax(-1)\n", - " precision, recall, f1, _ = precision_recall_fscore_support(\n", - " labels, preds, average=\"macro\"\n", - " )\n", - " acc = accuracy_score(labels, preds)\n", - "\n", - " if experiment:\n", - " epoch = int(experiment.curr_epoch) if experiment.curr_epoch is not None else 0\n", - " experiment.set_epoch(epoch)\n", - " experiment.log_confusion_matrix(\n", - " y_true=labels,\n", - " y_predicted=preds,\n", - " file_name=f\"confusion-matrix-epoch-{epoch}.json\",\n", - " labels=[\"negative\", \"postive\"],\n", - " index_to_example_function=get_example,\n", - " )\n", - "\n", - " return {\"accuracy\": acc, \"f1\": f1, \"precision\": precision, \"recall\": recall}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "JomdNlu0JORT" - }, - "source": [ - "# Run Training" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "7YIJF9DWKhe9" - }, - "source": [ - "In order to enable logging from the Hugging Face Trainer, you will need to set the `COMET_MODE` environment variable to `ONLINE`. If you would like to log assets produced in the training run as Comet Assets, set `COMET_LOG_ASSETS=TRUE` " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "kPmjwEH4NrQF" - }, - "outputs": [], - "source": [ - "%env COMET_MODE=ONLINE\n", - "%env COMET_LOG_ASSETS=TRUE\n", - "\n", - "training_args = TrainingArguments(\n", - " seed=SEED,\n", - " output_dir=\"./results\",\n", - " overwrite_output_dir=True,\n", - " num_train_epochs=1,\n", - " do_train=True,\n", - " do_eval=True,\n", - " evaluation_strategy=\"steps\",\n", - " eval_steps=25,\n", - " save_strategy=\"steps\",\n", - " save_total_limit=10,\n", - " save_steps=25,\n", - " per_device_train_batch_size=8,\n", - " report_to=[\"comet_ml\"],\n", - ")\n", - "\n", - "trainer = Trainer(\n", - " model=model,\n", - " args=training_args,\n", - " train_dataset=train_dataset,\n", - " eval_dataset=eval_dataset,\n", - " compute_metrics=compute_metrics,\n", - " data_collator=data_collator,\n", - ")\n", - "trainer.train()" - ] - } - ], - "metadata": { - "accelerator": "GPU", - "colab": { - "name": "Comet with Hugging Face", - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb b/integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb new file mode 100644 index 00000000..19a9afef --- /dev/null +++ b/integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb @@ -0,0 +1,902 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "UG0Hxf-nFKG2" + }, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5L-2VqFDWgGx" + }, + "source": [ + "[Hugging Face](https://huggingface.co/docs) is a community and data science platform that provides tools that enable users to build, train and deploy ML models based on open source (OS) code and technologies. Primarily known for their `transformers` library, Hugging Face has helped democratized access to these models by providing a unified API to train and evaluate a number of popular models for NLP.\n", + "\n", + "Comet integrates with Hugging Face's `Trainer` object, allowing you to log your model parameters, metrics, and assets such as model checkpoints. Learn more about our integration [here](https://www.comet.com/docs/v2/integrations/ml-frameworks/huggingface/)\n", + "\n", + "Curious about how Comet can help you build better models, faster? Find out more about [Comet](https://www.comet.com/site/products/ml-experiment-tracking/?utm_campaign=transformers&utm_medium=colab) and our [other integrations](https://www.comet.ml/docs/v2/integrations/overview/)\n", + "\n", + "\n", + "Get a preview for what's to come. Check out a completed experiment created from this notebook [here](https://www.comet.com/examples/comet-examples-transformers-trainer/3992ddee441f446bbb65c3cc4c8bd33b)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KUdcwLCWIz2n" + }, + "source": [ + "# Install Comet and Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "id": "nGDxYlCEKHMt" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: comet_ml in /home/lothiraldan/project/cometml/comet-python-client/comet-client-lib (3.42.1)\n", + "Requirement already satisfied: torch in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (2.3.0)\n", + "Requirement already satisfied: datasets in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (2.19.0)\n", + "Requirement already satisfied: transformers in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (4.41.0.dev0)\n", + "Requirement already satisfied: scikit-learn in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (1.4.2)\n", + "Requirement already satisfied: accelerate in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (0.29.3)\n", + "Requirement already satisfied: requests>=2.18.4 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (2.28.2)\n", + "Requirement already satisfied: requests-toolbelt>=0.8.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (0.10.1)\n", + "Requirement already satisfied: wurlitzer>=1.0.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (3.0.2)\n", + "Requirement already satisfied: everett[ini]<3.2.0,>=1.0.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (2.0.1)\n", + "Requirement already satisfied: jsonschema!=3.1.0,>=2.6.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (4.19.0)\n", + "Requirement already satisfied: wrapt>=1.11.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (1.13.3)\n", + "Requirement already satisfied: semantic-version>=2.8.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (2.8.5)\n", + "Requirement already satisfied: sentry-sdk>=1.1.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (1.45.0)\n", + "Requirement already satisfied: simplejson in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (3.18.0)\n", + "Requirement already satisfied: python-box<7.0.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (6.1.0)\n", + "Requirement already satisfied: urllib3>=1.21.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (1.26.16)\n", + "Requirement already satisfied: psutil>=5.6.3 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (5.9.4)\n", + "Requirement already satisfied: dulwich!=0.20.33,>=0.20.6 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (0.21.7)\n", + "Requirement already satisfied: rich>=13.3.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (13.3.5)\n", + "Requirement already satisfied: networkx in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (3.1)\n", + "Requirement already satisfied: jinja2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (3.1.2)\n", + "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (10.3.2.106)\n", + "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (11.4.5.107)\n", + "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.3.1)\n", + "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.0.106)\n", + "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (2.20.5)\n", + "Requirement already satisfied: triton==2.3.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (2.3.0)\n", + "Requirement already satisfied: fsspec in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (2024.2.0)\n", + "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (11.0.2.54)\n", + "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (8.9.2.26)\n", + "Requirement already satisfied: sympy in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (1.11.1)\n", + "Requirement already satisfied: typing-extensions>=4.8.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (4.8.0)\n", + "Requirement already satisfied: filelock in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (3.13.1)\n", + "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", + "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.3.101)\n", + "Requirement already satisfied: pyarrow>=12.0.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (12.0.1)\n", + "Requirement already satisfied: multiprocess in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.70.12.2)\n", + "Requirement already satisfied: aiohttp in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (3.8.4)\n", + "Requirement already satisfied: tqdm>=4.62.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (4.62.3)\n", + "Requirement already satisfied: packaging in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (23.2)\n", + "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.3.4)\n", + "Requirement already satisfied: numpy>=1.17 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (1.23.5)\n", + "Requirement already satisfied: pyyaml>=5.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (5.4.1)\n", + "Requirement already satisfied: pandas in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (2.0.3)\n", + "Requirement already satisfied: huggingface-hub>=0.21.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.23.0)\n", + "Requirement already satisfied: pyarrow-hotfix in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.6)\n", + "Requirement already satisfied: xxhash in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (3.1.0)\n", + "Requirement already satisfied: regex!=2019.12.17 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from transformers) (2022.9.13)\n", + "Requirement already satisfied: tokenizers<0.20,>=0.19 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from transformers) (0.19.1)\n", + "Requirement already satisfied: safetensors>=0.4.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from transformers) (0.4.2)\n", + "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from scikit-learn) (3.0.0)\n", + "Requirement already satisfied: scipy>=1.6.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from scikit-learn) (1.9.3)\n", + "Requirement already satisfied: joblib>=1.2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from scikit-learn) (1.2.0)\n", + "Requirement already satisfied: configobj in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from everett[ini]<3.2.0,>=1.0.1->comet_ml) (5.0.6)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (4.0.2)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (1.7.2)\n", + "Requirement already satisfied: attrs>=17.3.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (23.1.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (6.0.2)\n", + "Requirement already satisfied: frozenlist>=1.1.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (1.3.0)\n", + "Requirement already satisfied: aiosignal>=1.1.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (1.2.0)\n", + "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (2.0.9)\n", + "Requirement already satisfied: referencing>=0.28.4 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jsonschema!=3.1.0,>=2.6.0->comet_ml) (0.30.2)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jsonschema!=3.1.0,>=2.6.0->comet_ml) (0.10.3)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jsonschema!=3.1.0,>=2.6.0->comet_ml) (2023.7.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from requests>=2.18.4->comet_ml) (3.3)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from requests>=2.18.4->comet_ml) (2022.12.7)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from rich>=13.3.2->comet_ml) (2.17.2)\n", + "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from rich>=13.3.2->comet_ml) (2.2.0)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jinja2->torch) (2.1.3)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from pandas->datasets) (2.8.2)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from pandas->datasets) (2021.3)\n", + "Requirement already satisfied: tzdata>=2022.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from pandas->datasets) (2023.3)\n", + "Requirement already satisfied: mpmath>=0.19 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from sympy->torch) (1.3.0)\n", + "Requirement already satisfied: mdurl~=0.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=13.3.2->comet_ml) (0.1.2)\n", + "Requirement already satisfied: six>=1.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/home/lothiraldan/.virtualenvs/comet3.9/bin/python -m pip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install comet_ml torch datasets transformers scikit-learn accelerate" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3VaHie31I3r_" + }, + "source": [ + "# Initialize Comet" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "id": "8twVwYTIKK_x" + }, + "outputs": [], + "source": [ + "import comet_ml\n", + "\n", + "comet_ml.init(project_name=\"comet-examples-transfomers-trainer\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "y-isTv4sTU1n" + }, + "source": [ + "# Set Model Type" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "2R9ckDVbTWaN" + }, + "outputs": [], + "source": [ + "PRE_TRAINED_MODEL_NAME = \"distilbert/distilroberta-base\"\n", + "SEED = 42" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "dJRfuqCXI948" + }, + "source": [ + "# Load Data" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "id": "AD_Ejs6sKTRl" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-05-29 11:36:21.081337: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", + "2024-05-29 11:36:21.081382: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "2024-05-29 11:36:21.083078: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", + "2024-05-29 11:36:21.090918: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "2024-05-29 11:36:22.317986: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" + ] + } + ], + "source": [ + "from transformers import AutoTokenizer, Trainer, TrainingArguments\n", + "from datasets import load_dataset\n", + "\n", + "raw_datasets = load_dataset(\"imdb\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "IDXLkGRnI_jn" + }, + "source": [ + "# Setup Tokenizer" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "DI24QdbPKoO9" + }, + "outputs": [], + "source": [ + "tokenizer = AutoTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "flBnDDk4M8Yo" + }, + "outputs": [ + { + "data": { + "application/json": { + "ascii": false, + "bar_format": null, + "colour": null, + "elapsed": 0.007119655609130859, + "initial": 0, + "n": 0, + "ncols": null, + "nrows": 57, + "postfix": null, + "prefix": "Map", + "rate": null, + "total": 25000, + "unit": " examples", + "unit_divisor": 1000, + "unit_scale": false + }, + "application/vnd.jupyter.widget-view+json": { + "model_id": "4f507e248d7e4d4aa105759fea38236a", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Map: 0%| | 0/25000 [00:00\n", + " \n", + " \n", + " [25/25 07:04, Epoch 1/1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
StepTraining LossValidation LossAccuracyF1PrecisionRecall
25No log0.6287390.5950000.4949970.7810810.578125

" + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "TrainOutput(global_step=25, training_loss=0.6734719085693359, metrics={'train_runtime': 425.5504, 'train_samples_per_second': 0.47, 'train_steps_per_second': 0.059, 'total_flos': 26493479731200.0, 'train_loss': 0.6734719085693359, 'epoch': 1.0})" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "%env COMET_MODE=ONLINE\n", + "%env COMET_LOG_ASSETS=TRUE\n", + "\n", + "training_args = TrainingArguments(\n", + " seed=SEED,\n", + " output_dir=\"./results\",\n", + " overwrite_output_dir=True,\n", + " num_train_epochs=1,\n", + " do_train=True,\n", + " do_eval=True,\n", + " evaluation_strategy=\"steps\",\n", + " eval_steps=25,\n", + " save_strategy=\"steps\",\n", + " save_total_limit=10,\n", + " save_steps=25,\n", + " per_device_train_batch_size=8,\n", + " report_to=[\"comet_ml\"],\n", + ")\n", + "\n", + "trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " train_dataset=train_dataset,\n", + " eval_dataset=eval_dataset,\n", + " compute_metrics=compute_metrics,\n", + " data_collator=data_collator,\n", + ")\n", + "trainer.train()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Comet.ml Experiment Summary\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Data:\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m display_summary_level : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m name : registered_ampere_8049\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m url : https://www.comet.com/lothiraldan/comet-examples-transfomers-trainer/5308c4cebfff4a9db8b425b6515bbb55\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Metrics [count] (min, max):\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m epoch : 1.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_accuracy : 0.595\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_f1 : 0.4949967268306369\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_loss : 0.6287391781806946\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_precision : 0.7810810810810811\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_recall : 0.578125\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_runtime : 98.7791\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_samples_per_second : 2.025\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_steps_per_second : 0.253\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m loss [3] : (0.5943612456321716, 0.756797194480896)\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m total_flos : 26493479731200.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_loss : 0.6734719085693359\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_runtime : 425.5504\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_samples_per_second : 0.47\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_steps_per_second : 0.059\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Others:\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m hasNestedParams : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Parameters:\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|dispatch_batches : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|even_batches : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|gradient_accumulation_kwargs : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|non_blocking : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|split_batches : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|use_seedable_sampler : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adafactor : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adam_beta1 : 0.9\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adam_beta2 : 0.999\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adam_epsilon : 1e-08\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|auto_find_batch_size : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|batch_eval_metrics : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|bf16 : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|bf16_full_eval : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|data_seed : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_drop_last : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_num_workers : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_persistent_workers : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_pin_memory : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_prefetch_factor : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_backend : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_broadcast_buffers : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_bucket_cap_mb : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_find_unused_parameters : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_timeout : 1800\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|debug : []\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|deepspeed : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|disable_tqdm : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dispatch_batches : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|do_eval : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|do_predict : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|do_train : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_accumulation_steps : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_delay : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_do_concat_batches : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_steps : 25\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_strategy : steps\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|evaluation_strategy : steps\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16 : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16_backend : auto\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16_full_eval : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16_opt_level : O1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp : []\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|min_num_params : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|xla : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|xla_fsdp_grad_ckpt : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|xla_fsdp_v2 : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_min_num_params : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_transformer_layer_cls_to_wrap : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|full_determinism : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|gradient_accumulation_steps : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|gradient_checkpointing : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|gradient_checkpointing_kwargs : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|greater_is_better : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|group_by_length : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|half_precision_backend : auto\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_always_push : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_model_id : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_private_repo : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_strategy : every_save\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_token : \n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ignore_data_skip : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|include_inputs_for_metrics : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|include_num_input_tokens_seen : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|include_tokens_per_second : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|jit_mode_eval : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|label_names : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|label_smoothing_factor : 0.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|learning_rate : 5e-05\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|length_column_name : length\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|load_best_model_at_end : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|local_rank : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|log_level : passive\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|log_level_replica : warning\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|log_on_each_node : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_dir : ./results/runs/May29_11-37-14_fedora\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_first_step : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_nan_inf_filter : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_steps : 500\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_strategy : steps\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|lr_scheduler_kwargs : {}\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|lr_scheduler_type : linear\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|max_grad_norm : 1.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|max_steps : -1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|metric_for_best_model : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|mp_parameters : \n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|neftune_noise_alpha : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|no_cuda : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|num_train_epochs : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|optim : adamw_torch\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|optim_args : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|optim_target_modules : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|output_dir : ./results\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|overwrite_output_dir : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|past_index : -1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_device_eval_batch_size : 8\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_device_train_batch_size : 8\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_gpu_eval_batch_size : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_gpu_train_batch_size : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|prediction_loss_only : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub_model_id : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub_organization : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub_token : \n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ray_scope : last\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|remove_unused_columns : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|report_to : ['comet_ml']\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|restore_callback_states_from_checkpoint : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|resume_from_checkpoint : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|run_name : ./results\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_on_each_node : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_only_model : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_safetensors : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_steps : 25\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_strategy : steps\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_total_limit : 10\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|seed : 42\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|skip_memory_metrics : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|split_batches : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|tf32 : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torch_compile : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torch_compile_backend : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torch_compile_mode : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torchdynamo : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|tpu_metrics_debug : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|tpu_num_cores : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_cpu : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_ipex : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_legacy_prediction_loop : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_mps_device : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|warmup_ratio : 0.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|warmup_steps : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|weight_decay : 0.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|_name_or_path : distilbert/distilroberta-base\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|add_cross_attention : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|architectures : ['RobertaForMaskedLM']\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|attention_probs_dropout_prob : 0.1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|bad_words_ids : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|begin_suppress_tokens : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|bos_token_id : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|chunk_size_feed_forward : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|classifier_dropout : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|cross_attention_hidden_size : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|decoder_start_token_id : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|diversity_penalty : 0.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|do_sample : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|early_stopping : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|encoder_no_repeat_ngram_size : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|eos_token_id : 2\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|exponential_decay_length_penalty : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|finetuning_task : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|forced_bos_token_id : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|forced_eos_token_id : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|hidden_act : gelu\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|hidden_dropout_prob : 0.1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|hidden_size : 768\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|id2label|0 : LABEL_0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|id2label|1 : LABEL_1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|initializer_range : 0.02\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|intermediate_size : 3072\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|is_decoder : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|is_encoder_decoder : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|label2id|LABEL_0 : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|label2id|LABEL_1 : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|layer_norm_eps : 1e-05\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|length_penalty : 1.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|max_length : 20\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|max_position_embeddings : 514\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|min_length : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|model_type : roberta\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|no_repeat_ngram_size : 0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_attention_heads : 12\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_beam_groups : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_beams : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_hidden_layers : 6\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_return_sequences : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|output_attentions : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|output_hidden_states : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|output_scores : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|pad_token_id : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|position_embedding_type : absolute\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|prefix : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|problem_type : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|pruned_heads : {}\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|remove_invalid_values : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|repetition_penalty : 1.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|return_dict : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|return_dict_in_generate : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|sep_token_id : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|suppress_tokens : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|task_specific_params : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|temperature : 1.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tf_legacy_loss : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tie_encoder_decoder : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tie_word_embeddings : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tokenizer_class : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|top_k : 50\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|top_p : 1.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|torch_dtype : None\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|torchscript : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|transformers_version : 4.41.0.dev0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|type_vocab_size : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|typical_p : 1.0\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|use_bfloat16 : False\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|use_cache : True\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|vocab_size : 50265\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Uploads:\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m asset : 49 (5.41 GB)\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m confusion-matrix : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m environment details : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m filename : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m git metadata : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m git-patch (uncompressed) : 1 (404.82 KB)\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m installed packages : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m model graph : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m notebook : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m source_code : 1\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m \n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Please wait for assets to finish uploading (timeout is 10800 seconds)\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 12 file(s), remaining 3.26 GB/4.91 GB\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 9 asset(s), remaining 2.20 GB/4.16 GB, Throughput 72.32 MB/s, ETA ~32s\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 8 asset(s), remaining 1.25 GB/3.91 GB, Throughput 65.02 MB/s, ETA ~20s\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 4 asset(s), remaining 544.33 MB/2.11 GB, Throughput 48.95 MB/s, ETA ~12s\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 2 asset(s), remaining 162.51 MB/1.11 GB, Throughput 25.42 MB/s, ETA ~7s\n", + "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 1 asset(s), remaining 20.91 MB/510.91 MB, Throughput 9.43 MB/s, ETA ~3s\n" + ] + } + ], + "source": [ + "comet_ml.get_running_experiment().end()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "name": "Comet with Hugging Face", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/README.md b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/README.md similarity index 100% rename from integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/README.md rename to integrations/model-training/transformers/transformers-distilbert-fine-tuning/README.md diff --git a/integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/data/title_conference.csv b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/data/title_conference.csv similarity index 100% rename from integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/data/title_conference.csv rename to integrations/model-training/transformers/transformers-distilbert-fine-tuning/data/title_conference.csv diff --git a/integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/requirements.txt b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt similarity index 69% rename from integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/requirements.txt rename to integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt index ff7b74dc..c2a47abc 100644 --- a/integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/requirements.txt +++ b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt @@ -3,4 +3,4 @@ comet_ml pandas scikit-learn torch -transformers<4.40.0 +transformers diff --git a/integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py similarity index 100% rename from integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py rename to integrations/model-training/transformers/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py From 2f7651fa14f6a7a118696a1165c8fb1322b49119 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 18 Jun 2024 17:37:06 +0200 Subject: [PATCH 2/8] Add new example based on more modern example --- .../requirements.txt | 7 ++ .../transformers-google-bert-fine-tuning.py | 78 +++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt create mode 100644 integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py diff --git a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt new file mode 100644 index 00000000..7e9dadc1 --- /dev/null +++ b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt @@ -0,0 +1,7 @@ +comet_ml +datasets +evaluate +numpy +scikit-learn +torch +transformers[torch] diff --git a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py new file mode 100644 index 00000000..64096319 --- /dev/null +++ b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py @@ -0,0 +1,78 @@ +# coding: utf-8 +import comet_ml + +import evaluate +import numpy as np +from datasets import load_dataset + +from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + Trainer, + TrainingArguments, + enable_full_determinism, +) + +SEED = 42 + +enable_full_determinism(SEED) + +# Login to Comet if needed +comet_ml.init(project_name="comet-example-transformers-google-bert-fine-tuning") + + +tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-cased") + + +def tokenize_function(examples): + return tokenizer(examples["text"], padding="max_length", truncation=True) + + +dataset = load_dataset("yelp_review_full") +dataset["train"] = dataset["train"].shuffle(seed=SEED).select(range(100)) +dataset["test"] = dataset["test"].shuffle(seed=SEED).select(range(100)) + +tokenized_datasets = dataset.map(tokenize_function, batched=True) + +small_train_dataset = tokenized_datasets["train"] +small_eval_dataset = tokenized_datasets["test"] + +model = AutoModelForSequenceClassification.from_pretrained( + "google-bert/bert-base-cased", num_labels=5 +) + +metric = evaluate.load("accuracy") + + +def compute_metrics(eval_pred): + logits, labels = eval_pred + predictions = np.argmax(logits, axis=-1) + + experiment = comet_ml.get_running_experiment() + if experiment: + experiment.log_confusion_matrix(predictions, labels) + + return metric.compute(predictions=predictions, references=labels) + + +EPOCHS = 5 + +training_args = TrainingArguments( + seed=SEED, + output_dir="./results", + overwrite_output_dir=True, + num_train_epochs=EPOCHS, + eval_strategy="epoch", + do_train=True, + do_eval=True, + report_to=["all"], +) +trainer = Trainer( + model=model, + args=training_args, + train_dataset=small_train_dataset, + eval_dataset=small_eval_dataset, + compute_metrics=compute_metrics, +) + +trainer.train() From 45eeb293f5117ff36c519d56bf81c8e4c6a9f797 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 18 Jun 2024 17:37:55 +0200 Subject: [PATCH 3/8] Update README link --- transformers/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transformers/README.md b/transformers/README.md index 5b18a8cf..835dd5df 100644 --- a/transformers/README.md +++ b/transformers/README.md @@ -1 +1 @@ -All of the Transformers examples has been moved here: https://github.com/comet-ml/comet-examples/tree/master/integrations/model-training/hugging_face/. +All of the Transformers examples has been moved here: https://github.com/comet-ml/comet-examples/tree/master/integrations/model-training/transformers/. From b27353478fce6e4abf958f318584d595cae84455 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 18 Jun 2024 18:51:20 +0200 Subject: [PATCH 4/8] Add Ray Train+Transformers example --- ...h_ray_train_huggingface_transformers.ipynb | 313 ++++++++++++++++++ 1 file changed, 313 insertions(+) create mode 100644 integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb diff --git a/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb new file mode 100644 index 00000000..a1511301 --- /dev/null +++ b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb @@ -0,0 +1,313 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[Comet](https://www.comet.com/site/products/ml-experiment-tracking/?utm_campaign=ray_train&utm_medium=colab) is an MLOps Platform that is designed to help Data Scientists and Teams build better models faster! Comet provides tooling to track, Explain, Manage, and Monitor your models in a single place! It works with Jupyter Notebooks and Scripts and most importantly it's 100% free to get started!\n", + "\n", + "[Ray Train](https://docs.ray.io/en/latest/train/train.html) abstracts away the complexity of setting up a distributed training system.\n", + "\n", + "Instrument your runs with Comet to start managing experiments, create dataset versions and track hyperparameters for faster and easier reproducibility and collaboration.\n", + "\n", + "[Find more information about our integration with Ray Train](https://www.comet.ml/docs/v2/integrations/ml-frameworks/ray/)\n", + "\n", + "Get a preview for what's to come. Check out a completed experiment created from this notebook [here](https://www.comet.com/examples/comet-example-ray-train-keras/99d169308c854be7ac222c995a2bfa26?experiment-tab=systemMetrics).\n", + "\n", + "This example is based on the [following Ray Train Tensorflow example](https://docs.ray.io/en/latest/train/examples/tf/tensorflow_mnist_example.html)." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZYchV5RWwdv5" + }, + "source": [ + "# Install Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "DJnmqphuY2eI" + }, + "outputs": [], + "source": [ + "%pip install comet_ml \"ray[air]>=2.1.0\" \"transformers\" \"accelerate>=0.12.0\" \"datasets\" \"sentencepiece\" scipy \"scikit-learn\" protobuf \"torch>=1.3\" evaluate" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "crOcPHobwhGL" + }, + "source": [ + "# Initialize Comet" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "HNQRM0U3caiY" + }, + "outputs": [], + "source": [ + "import comet_ml\n", + "import comet_ml.integration.ray\n", + "\n", + "comet_ml.init()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cgqwGSwtzVWD" + }, + "source": [ + "# Import Dependencies" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "e-5rRYaUw5AF" + }, + "outputs": [], + "source": [ + "import os\n", + "import threading\n", + "\n", + "import evaluate\n", + "import numpy as np\n", + "from datasets import load_dataset\n", + "\n", + "from transformers import (\n", + " AutoModelForSequenceClassification,\n", + " AutoTokenizer,\n", + " Trainer,\n", + " TrainingArguments,\n", + " enable_full_determinism,\n", + ")\n", + "\n", + "import ray.train.huggingface.transformers\n", + "from ray.train import ScalingConfig, RunConfig\n", + "from ray.train.torch import TorchTrainer" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Models\n", + "PRE_TRAINED_MODEL_NAME = \"google-bert/bert-base-cased\"\n", + "SEED = 42\n", + "\n", + "enable_full_determinism(SEED)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prepare your dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def get_dataset():\n", + " tokenizer = AutoTokenizer.from_pretrained(\"google-bert/bert-base-cased\")\n", + "\n", + " def tokenize_function(examples):\n", + " return tokenizer(examples[\"text\"], padding=\"max_length\", truncation=True)\n", + "\n", + " dataset = load_dataset(\"yelp_review_full\")\n", + " dataset[\"train\"] = dataset[\"train\"].shuffle(seed=SEED).select(range(100))\n", + " dataset[\"test\"] = dataset[\"test\"].shuffle(seed=SEED).select(range(100))\n", + "\n", + " tokenized_datasets = dataset.map(tokenize_function, batched=True)\n", + "\n", + " small_train_dataset = tokenized_datasets[\"train\"]\n", + " small_eval_dataset = tokenized_datasets[\"test\"]\n", + " return (small_train_dataset, small_eval_dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TJuThf1TxP_G" + }, + "source": [ + "# Define your distributed training function\n", + "\n", + "This function is gonna be distributed and executed on each distributed worker." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def train_func(config):\n", + " from comet_ml import get_running_experiment\n", + " from comet_ml.integration.ray import comet_worker_logger\n", + "\n", + " with comet_worker_logger(config) as experiment:\n", + " small_train_dataset, small_eval_dataset = get_dataset()\n", + "\n", + " # Model\n", + " model = AutoModelForSequenceClassification.from_pretrained(\n", + " \"google-bert/bert-base-cased\", num_labels=5\n", + " )\n", + "\n", + " # Evaluation Metrics\n", + " metric = evaluate.load(\"accuracy\")\n", + "\n", + " def compute_metrics(eval_pred):\n", + " print(\"COMPUTE METRICS CALLED!\")\n", + " logits, labels = eval_pred\n", + " predictions = np.argmax(logits, axis=-1)\n", + "\n", + " experiment = comet_ml.get_running_experiment()\n", + " if experiment:\n", + " experiment.log_confusion_matrix(predictions, labels)\n", + "\n", + " return metric.compute(predictions=predictions, references=labels)\n", + "\n", + " # Hugging Face Trainer\n", + " training_args = TrainingArguments(\n", + " seed=SEED,\n", + " output_dir=\"./results\",\n", + " overwrite_output_dir=True,\n", + " num_train_epochs=config[\"epochs\"],\n", + " eval_strategy=\"epoch\",\n", + " do_train=True,\n", + " do_eval=True,\n", + " report_to=[\"comet_ml\"],\n", + " )\n", + " trainer = Trainer(\n", + " model=model,\n", + " args=training_args,\n", + " train_dataset=small_train_dataset,\n", + " eval_dataset=small_eval_dataset,\n", + " compute_metrics=compute_metrics,\n", + " )\n", + "\n", + " # Report Metrics and Checkpoints to Ray Train\n", + " callback = ray.train.huggingface.transformers.RayTrainReportCallback()\n", + " trainer.add_callback(callback)\n", + "\n", + " # Prepare Transformers Trainer\n", + " trainer = ray.train.huggingface.transformers.prepare_trainer(trainer)\n", + "\n", + " # Start Training\n", + " trainer.train()\n", + "\n", + " comet_ml.get_running_experiment().end()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Define the function that schedule the distributed job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def train(num_workers: int = 2, use_gpu: bool = False, epochs=1):\n", + " scaling_config = ScalingConfig(num_workers=num_workers, use_gpu=use_gpu)\n", + " config = {\"use_gpu\": use_gpu, \"epochs\": 2}\n", + "\n", + " callback = comet_ml.integration.ray.CometTrainLoggerCallback(\n", + " config, project_name=\"comet-example-ray-train-hugginface-transformers\"\n", + " )\n", + "\n", + " ray_trainer = TorchTrainer(\n", + " train_func,\n", + " scaling_config=scaling_config,\n", + " train_loop_config=config,\n", + " run_config=RunConfig(callbacks=[callback]),\n", + " )\n", + " result = ray_trainer.fit()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Train the model\n", + "\n", + "Ray will wait indefinitely if we request more num_workers that the available resources, the code below ensure we never request more CPU than available locally." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ideal_num_workers = 2\n", + "\n", + "available_local_cpu_count = os.cpu_count() - 1\n", + "num_workers = min(ideal_num_workers, available_local_cpu_count)\n", + "\n", + "if num_workers < 1:\n", + " num_workers = 1\n", + "\n", + "train(num_workers, use_gpu=False, epochs=5)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From f3def71910e3cad0bd38b1d95730704494709a17 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Tue, 2 Jul 2024 16:59:23 +0200 Subject: [PATCH 5/8] Set correct minimum Comet version --- .../Comet_with_Hugging_Face_Trainer.ipynb | 573 +----------------- .../requirements.txt | 2 +- .../requirements.txt | 2 +- .../transformers-google-bert-fine-tuning.py | 2 +- 4 files changed, 22 insertions(+), 557 deletions(-) diff --git a/integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb b/integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb index 19a9afef..c02b4b2b 100644 --- a/integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb +++ b/integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb @@ -36,103 +36,13 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "id": "nGDxYlCEKHMt" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: comet_ml in /home/lothiraldan/project/cometml/comet-python-client/comet-client-lib (3.42.1)\n", - "Requirement already satisfied: torch in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (2.3.0)\n", - "Requirement already satisfied: datasets in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (2.19.0)\n", - "Requirement already satisfied: transformers in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (4.41.0.dev0)\n", - "Requirement already satisfied: scikit-learn in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (1.4.2)\n", - "Requirement already satisfied: accelerate in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (0.29.3)\n", - "Requirement already satisfied: requests>=2.18.4 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (2.28.2)\n", - "Requirement already satisfied: requests-toolbelt>=0.8.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (0.10.1)\n", - "Requirement already satisfied: wurlitzer>=1.0.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (3.0.2)\n", - "Requirement already satisfied: everett[ini]<3.2.0,>=1.0.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (2.0.1)\n", - "Requirement already satisfied: jsonschema!=3.1.0,>=2.6.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (4.19.0)\n", - "Requirement already satisfied: wrapt>=1.11.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (1.13.3)\n", - "Requirement already satisfied: semantic-version>=2.8.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (2.8.5)\n", - "Requirement already satisfied: sentry-sdk>=1.1.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (1.45.0)\n", - "Requirement already satisfied: simplejson in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (3.18.0)\n", - "Requirement already satisfied: python-box<7.0.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (6.1.0)\n", - "Requirement already satisfied: urllib3>=1.21.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (1.26.16)\n", - "Requirement already satisfied: psutil>=5.6.3 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (5.9.4)\n", - "Requirement already satisfied: dulwich!=0.20.33,>=0.20.6 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (0.21.7)\n", - "Requirement already satisfied: rich>=13.3.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from comet_ml) (13.3.5)\n", - "Requirement already satisfied: networkx in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (3.1)\n", - "Requirement already satisfied: jinja2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (3.1.2)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.20.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (2.20.5)\n", - "Requirement already satisfied: triton==2.3.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (2.3.0)\n", - "Requirement already satisfied: fsspec in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (2024.2.0)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (8.9.2.26)\n", - "Requirement already satisfied: sympy in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (1.11.1)\n", - "Requirement already satisfied: typing-extensions>=4.8.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (4.8.0)\n", - "Requirement already satisfied: filelock in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (3.13.1)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from torch) (12.1.105)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.3.101)\n", - "Requirement already satisfied: pyarrow>=12.0.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (12.0.1)\n", - "Requirement already satisfied: multiprocess in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.70.12.2)\n", - "Requirement already satisfied: aiohttp in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (3.8.4)\n", - "Requirement already satisfied: tqdm>=4.62.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (4.62.3)\n", - "Requirement already satisfied: packaging in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (23.2)\n", - "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.3.4)\n", - "Requirement already satisfied: numpy>=1.17 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (1.23.5)\n", - "Requirement already satisfied: pyyaml>=5.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (5.4.1)\n", - "Requirement already satisfied: pandas in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (2.0.3)\n", - "Requirement already satisfied: huggingface-hub>=0.21.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.23.0)\n", - "Requirement already satisfied: pyarrow-hotfix in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (0.6)\n", - "Requirement already satisfied: xxhash in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from datasets) (3.1.0)\n", - "Requirement already satisfied: regex!=2019.12.17 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from transformers) (2022.9.13)\n", - "Requirement already satisfied: tokenizers<0.20,>=0.19 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from transformers) (0.19.1)\n", - "Requirement already satisfied: safetensors>=0.4.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from transformers) (0.4.2)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from scikit-learn) (3.0.0)\n", - "Requirement already satisfied: scipy>=1.6.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from scikit-learn) (1.9.3)\n", - "Requirement already satisfied: joblib>=1.2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from scikit-learn) (1.2.0)\n", - "Requirement already satisfied: configobj in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from everett[ini]<3.2.0,>=1.0.1->comet_ml) (5.0.6)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (4.0.2)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (1.7.2)\n", - "Requirement already satisfied: attrs>=17.3.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (23.1.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (6.0.2)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (1.3.0)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (1.2.0)\n", - "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from aiohttp->datasets) (2.0.9)\n", - "Requirement already satisfied: referencing>=0.28.4 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jsonschema!=3.1.0,>=2.6.0->comet_ml) (0.30.2)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jsonschema!=3.1.0,>=2.6.0->comet_ml) (0.10.3)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jsonschema!=3.1.0,>=2.6.0->comet_ml) (2023.7.1)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from requests>=2.18.4->comet_ml) (3.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from requests>=2.18.4->comet_ml) (2022.12.7)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from rich>=13.3.2->comet_ml) (2.17.2)\n", - "Requirement already satisfied: markdown-it-py<3.0.0,>=2.2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from rich>=13.3.2->comet_ml) (2.2.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from jinja2->torch) (2.1.3)\n", - "Requirement already satisfied: python-dateutil>=2.8.2 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from pandas->datasets) (2.8.2)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from pandas->datasets) (2021.3)\n", - "Requirement already satisfied: tzdata>=2022.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from pandas->datasets) (2023.3)\n", - "Requirement already satisfied: mpmath>=0.19 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from sympy->torch) (1.3.0)\n", - "Requirement already satisfied: mdurl~=0.1 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from markdown-it-py<3.0.0,>=2.2.0->rich>=13.3.2->comet_ml) (0.1.2)\n", - "Requirement already satisfied: six>=1.5 in /home/lothiraldan/.virtualenvs/comet3.9/lib/python3.9/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip available: \u001b[0m\u001b[31;49m22.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49m/home/lothiraldan/.virtualenvs/comet3.9/bin/python -m pip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "outputs": [], "source": [ - "%pip install comet_ml torch datasets transformers scikit-learn accelerate" + "%pip install \"comet_ml>=3.43.2\" torch datasets transformers scikit-learn accelerate" ] }, { @@ -146,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "id": "8twVwYTIKK_x" }, @@ -168,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "id": "2R9ckDVbTWaN" }, @@ -189,24 +99,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "id": "AD_Ejs6sKTRl" }, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2024-05-29 11:36:21.081337: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "2024-05-29 11:36:21.081382: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", - "2024-05-29 11:36:21.083078: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "2024-05-29 11:36:21.090918: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", - "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", - "2024-05-29 11:36:22.317986: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n" - ] - } - ], + "outputs": [], "source": [ "from transformers import AutoTokenizer, Trainer, TrainingArguments\n", "from datasets import load_dataset\n", @@ -225,7 +122,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "id": "DI24QdbPKoO9" }, @@ -236,105 +133,11 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "id": "flBnDDk4M8Yo" }, - "outputs": [ - { - "data": { - "application/json": { - "ascii": false, - "bar_format": null, - "colour": null, - "elapsed": 0.007119655609130859, - "initial": 0, - "n": 0, - "ncols": null, - "nrows": 57, - "postfix": null, - "prefix": "Map", - "rate": null, - "total": 25000, - "unit": " examples", - "unit_divisor": 1000, - "unit_scale": false - }, - "application/vnd.jupyter.widget-view+json": { - "model_id": "4f507e248d7e4d4aa105759fea38236a", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Map: 0%| | 0/25000 [00:00\n", - " \n", - " \n", - " [25/25 07:04, Epoch 1/1]\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
StepTraining LossValidation LossAccuracyF1PrecisionRecall
25No log0.6287390.5950000.4949970.7810810.578125

" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "TrainOutput(global_step=25, training_loss=0.6734719085693359, metrics={'train_runtime': 425.5504, 'train_samples_per_second': 0.47, 'train_steps_per_second': 0.059, 'total_flos': 26493479731200.0, 'train_loss': 0.6734719085693359, 'epoch': 1.0})" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "%env COMET_MODE=ONLINE\n", "%env COMET_LOG_ASSETS=TRUE\n", @@ -600,267 +323,9 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Comet.ml Experiment Summary\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m ---------------------------------------------------------------------------------------\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Data:\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m display_summary_level : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m name : registered_ampere_8049\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m url : https://www.comet.com/lothiraldan/comet-examples-transfomers-trainer/5308c4cebfff4a9db8b425b6515bbb55\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Metrics [count] (min, max):\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m epoch : 1.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_accuracy : 0.595\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_f1 : 0.4949967268306369\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_loss : 0.6287391781806946\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_precision : 0.7810810810810811\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_recall : 0.578125\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_runtime : 98.7791\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_samples_per_second : 2.025\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m eval_steps_per_second : 0.253\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m loss [3] : (0.5943612456321716, 0.756797194480896)\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m total_flos : 26493479731200.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_loss : 0.6734719085693359\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_runtime : 425.5504\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_samples_per_second : 0.47\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m train_steps_per_second : 0.059\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Others:\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m hasNestedParams : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Parameters:\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|dispatch_batches : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|even_batches : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|gradient_accumulation_kwargs : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|non_blocking : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|split_batches : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|accelerator_config|use_seedable_sampler : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adafactor : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adam_beta1 : 0.9\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adam_beta2 : 0.999\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|adam_epsilon : 1e-08\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|auto_find_batch_size : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|batch_eval_metrics : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|bf16 : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|bf16_full_eval : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|data_seed : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_drop_last : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_num_workers : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_persistent_workers : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_pin_memory : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dataloader_prefetch_factor : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_backend : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_broadcast_buffers : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_bucket_cap_mb : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_find_unused_parameters : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ddp_timeout : 1800\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|debug : []\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|deepspeed : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|disable_tqdm : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|dispatch_batches : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|do_eval : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|do_predict : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|do_train : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_accumulation_steps : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_delay : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_do_concat_batches : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_steps : 25\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|eval_strategy : steps\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|evaluation_strategy : steps\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16 : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16_backend : auto\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16_full_eval : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fp16_opt_level : O1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp : []\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|min_num_params : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|xla : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|xla_fsdp_grad_ckpt : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_config|xla_fsdp_v2 : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_min_num_params : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|fsdp_transformer_layer_cls_to_wrap : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|full_determinism : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|gradient_accumulation_steps : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|gradient_checkpointing : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|gradient_checkpointing_kwargs : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|greater_is_better : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|group_by_length : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|half_precision_backend : auto\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_always_push : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_model_id : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_private_repo : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_strategy : every_save\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|hub_token : \n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ignore_data_skip : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|include_inputs_for_metrics : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|include_num_input_tokens_seen : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|include_tokens_per_second : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|jit_mode_eval : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|label_names : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|label_smoothing_factor : 0.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|learning_rate : 5e-05\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|length_column_name : length\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|load_best_model_at_end : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|local_rank : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|log_level : passive\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|log_level_replica : warning\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|log_on_each_node : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_dir : ./results/runs/May29_11-37-14_fedora\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_first_step : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_nan_inf_filter : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_steps : 500\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|logging_strategy : steps\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|lr_scheduler_kwargs : {}\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|lr_scheduler_type : linear\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|max_grad_norm : 1.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|max_steps : -1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|metric_for_best_model : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|mp_parameters : \n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|neftune_noise_alpha : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|no_cuda : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|num_train_epochs : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|optim : adamw_torch\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|optim_args : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|optim_target_modules : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|output_dir : ./results\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|overwrite_output_dir : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|past_index : -1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_device_eval_batch_size : 8\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_device_train_batch_size : 8\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_gpu_eval_batch_size : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|per_gpu_train_batch_size : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|prediction_loss_only : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub_model_id : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub_organization : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|push_to_hub_token : \n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|ray_scope : last\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|remove_unused_columns : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|report_to : ['comet_ml']\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|restore_callback_states_from_checkpoint : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|resume_from_checkpoint : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|run_name : ./results\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_on_each_node : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_only_model : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_safetensors : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_steps : 25\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_strategy : steps\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|save_total_limit : 10\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|seed : 42\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|skip_memory_metrics : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|split_batches : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|tf32 : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torch_compile : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torch_compile_backend : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torch_compile_mode : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|torchdynamo : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|tpu_metrics_debug : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|tpu_num_cores : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_cpu : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_ipex : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_legacy_prediction_loop : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|use_mps_device : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|warmup_ratio : 0.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|warmup_steps : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m args|weight_decay : 0.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|_name_or_path : distilbert/distilroberta-base\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|add_cross_attention : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|architectures : ['RobertaForMaskedLM']\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|attention_probs_dropout_prob : 0.1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|bad_words_ids : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|begin_suppress_tokens : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|bos_token_id : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|chunk_size_feed_forward : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|classifier_dropout : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|cross_attention_hidden_size : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|decoder_start_token_id : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|diversity_penalty : 0.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|do_sample : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|early_stopping : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|encoder_no_repeat_ngram_size : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|eos_token_id : 2\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|exponential_decay_length_penalty : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|finetuning_task : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|forced_bos_token_id : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|forced_eos_token_id : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|hidden_act : gelu\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|hidden_dropout_prob : 0.1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|hidden_size : 768\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|id2label|0 : LABEL_0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|id2label|1 : LABEL_1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|initializer_range : 0.02\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|intermediate_size : 3072\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|is_decoder : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|is_encoder_decoder : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|label2id|LABEL_0 : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|label2id|LABEL_1 : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|layer_norm_eps : 1e-05\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|length_penalty : 1.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|max_length : 20\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|max_position_embeddings : 514\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|min_length : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|model_type : roberta\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|no_repeat_ngram_size : 0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_attention_heads : 12\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_beam_groups : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_beams : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_hidden_layers : 6\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|num_return_sequences : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|output_attentions : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|output_hidden_states : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|output_scores : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|pad_token_id : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|position_embedding_type : absolute\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|prefix : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|problem_type : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|pruned_heads : {}\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|remove_invalid_values : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|repetition_penalty : 1.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|return_dict : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|return_dict_in_generate : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|sep_token_id : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|suppress_tokens : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|task_specific_params : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|temperature : 1.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tf_legacy_loss : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tie_encoder_decoder : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tie_word_embeddings : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|tokenizer_class : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|top_k : 50\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|top_p : 1.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|torch_dtype : None\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|torchscript : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|transformers_version : 4.41.0.dev0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|type_vocab_size : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|typical_p : 1.0\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|use_bfloat16 : False\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|use_cache : True\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m config|vocab_size : 50265\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Uploads:\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m asset : 49 (5.41 GB)\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m confusion-matrix : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m environment details : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m filename : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m git metadata : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m git-patch (uncompressed) : 1 (404.82 KB)\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m installed packages : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m model graph : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m notebook : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m source_code : 1\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m \n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Please wait for assets to finish uploading (timeout is 10800 seconds)\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 12 file(s), remaining 3.26 GB/4.91 GB\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 9 asset(s), remaining 2.20 GB/4.16 GB, Throughput 72.32 MB/s, ETA ~32s\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 8 asset(s), remaining 1.25 GB/3.91 GB, Throughput 65.02 MB/s, ETA ~20s\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 4 asset(s), remaining 544.33 MB/2.11 GB, Throughput 48.95 MB/s, ETA ~12s\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 2 asset(s), remaining 162.51 MB/1.11 GB, Throughput 25.42 MB/s, ETA ~7s\n", - "\u001b[1;38;5;39mCOMET INFO:\u001b[0m Still uploading 1 asset(s), remaining 20.91 MB/510.91 MB, Throughput 9.43 MB/s, ETA ~3s\n" - ] - } - ], + "outputs": [], "source": [ "comet_ml.get_running_experiment().end()" ] diff --git a/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt index c2a47abc..c3efe123 100644 --- a/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt +++ b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt @@ -1,5 +1,5 @@ accelerate -comet_ml +comet_ml>=3.43.2 pandas scikit-learn torch diff --git a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt index 7e9dadc1..722db500 100644 --- a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt +++ b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt @@ -1,4 +1,4 @@ -comet_ml +comet_ml>=3.43.2 datasets evaluate numpy diff --git a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py index 64096319..c9d856e2 100644 --- a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py +++ b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py @@ -55,7 +55,7 @@ def compute_metrics(eval_pred): return metric.compute(predictions=predictions, references=labels) -EPOCHS = 5 +EPOCHS = 3 training_args = TrainingArguments( seed=SEED, From 9d349ffb366333d9aae8640f7d844d018b896578 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Thu, 25 Jul 2024 11:31:47 +0200 Subject: [PATCH 6/8] Fix the test matrix --- .github/workflows/test-examples.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-examples.yml b/.github/workflows/test-examples.yml index dba6a8da..b6a7694d 100644 --- a/.github/workflows/test-examples.yml +++ b/.github/workflows/test-examples.yml @@ -23,7 +23,6 @@ jobs: - integrations/model-optimization/ray-tune/notebooks/Comet_and_Ray.ipynb - integrations/model-training/composer/notebooks/comet_composer.ipynb - integrations/model-training/fastai/notebooks/fastai_hello_world.ipynb - - integrations/model-training/hugging_face/notebooks/Comet_with_Hugging_Face_Trainer.ipynb - integrations/model-training/keras/notebooks/Comet_with_Keras.ipynb - integrations/model-training/lightgbm/notebooks/Comet_and_LightGBM.ipynb - integrations/model-training/prophet/notebooks/Comet_and_Prophet.ipynb @@ -33,9 +32,11 @@ jobs: - integrations/model-training/pytorch/notebooks/Comet_Pytorch_Tensorboard.ipynb - integrations/model-training/pytorch/notebooks/Comet_Pytorch_TensorboardX.ipynb - integrations/model-training/pytorch/notebooks/Histogram_Logging_Pytorch.ipynb + - integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb - integrations/model-training/ray-train/notebooks/Comet_with_ray_train_keras.ipynb - integrations/model-training/ray-train/notebooks/Comet_with_ray_train_xgboost.ipynb - integrations/model-training/tensorflow/notebooks/Comet_and_Tensorflow.ipynb + - integrations/model-training/transformers/notebooks/Comet_with_Hugging_Face_Trainer.ipynb - integrations/model-training/yolov5/notebooks/Comet_and_YOLOv5.ipynb - integrations/model-training/yolov8/notebooks/YOLOv8_and_Comet.ipynb - integrations/reinforcement-learning/gymnasium/notebooks/comet_gymnasium_example.ipynb @@ -107,7 +108,6 @@ jobs: - {script: "integrations/model-optimization/optuna/optuna-hello-world/optuna-hello-world.py", arg: ""} - {script: "integrations/model-training/composer/mosaicml-getting-started/mosaicml-getting-started.py", arg: ""} - {script: "integrations/model-training/fastai/fastai-hello-world/fastai_hello_world.py", arg: ""} - - {script: "integrations/model-training/hugging_face/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py", arg: ""} - {script: "integrations/model-training/keras/keras-mnist-dnn/keras-mnist-dnn.py", arg: ""} - {script: "integrations/model-training/mlflow/mlflow-hello-world/mlflow-hello-world.py", arg: "run"} - {script: "integrations/model-training/pytorch-lightning/pytorch-lightning-optimizer/pytorch-lightning-optimizer.py", arg: ""} @@ -116,6 +116,8 @@ jobs: - {script: "integrations/model-training/pytorch/pytorch-tensorboard/pytorch-tensorboard-example.py", arg: ""} - {script: "integrations/model-training/scikit-learn/sklearn-classification-example/comet-scikit-classification-example.py", arg: "run"} - {script: "integrations/model-training/scikit-learn/sklearn-nlp-example/comet-scikit-nlp-example.py", args: ""} + - {script: "integrations/model-training/transformers/transformers-distilbert-fine-tuning/transformers-distilbert-fine-tuning.py", arg: ""} + - {script: "integrations/model-training/transformers/transformers-google-bert-fine-tuning/transformers-google-bert-fine-tuning.py", arg: ""} - {script: "integrations/model-training/xgboost/xgboost-california/xgboost-california.py", arg: ""} - {script: "integrations/workflow-orchestration/metaflow/metaflow-hello-world/helloworld.py", arg: "run"} - {script: "integrations/workflow-orchestration/metaflow/metaflow-model-evaluation/metaflow-model-evaluation.py", arg: "run --max-workers 1 --n_samples 100"} From 95f3fbf6ddfff3e92266031c51b796ae5436ab01 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Thu, 25 Jul 2024 15:04:23 +0200 Subject: [PATCH 7/8] Try smaller batch size to reduce CPU memory pressure on CI --- ...et_with_ray_train_huggingface_transformers.ipynb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb index a1511301..514a6ab4 100644 --- a/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb +++ b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb @@ -180,7 +180,6 @@ " metric = evaluate.load(\"accuracy\")\n", "\n", " def compute_metrics(eval_pred):\n", - " print(\"COMPUTE METRICS CALLED!\")\n", " logits, labels = eval_pred\n", " predictions = np.argmax(logits, axis=-1)\n", "\n", @@ -192,14 +191,16 @@ "\n", " # Hugging Face Trainer\n", " training_args = TrainingArguments(\n", - " seed=SEED,\n", + " do_eval=True,\n", + " do_train=True,\n", + " eval_strategy=\"epoch\",\n", + " num_train_epochs=config[\"epochs\"],\n", " output_dir=\"./results\",\n", " overwrite_output_dir=True,\n", - " num_train_epochs=config[\"epochs\"],\n", - " eval_strategy=\"epoch\",\n", - " do_train=True,\n", - " do_eval=True,\n", + " per_device_eval_batch_size=4,\n", + " per_device_train_batch_size=4,\n", " report_to=[\"comet_ml\"],\n", + " seed=SEED,\n", " )\n", " trainer = Trainer(\n", " model=model,\n", From d22cbdb8f091b21761408527d1751f18db8bbdf0 Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Thu, 25 Jul 2024 17:49:40 +0200 Subject: [PATCH 8/8] Fix last links --- ...h_ray_train_huggingface_transformers.ipynb | 2 +- .../README.md | 2 +- .../requirements.txt | 2 +- .../README.md | 37 +++++++++++++++++++ .../requirements.txt | 2 +- 5 files changed, 41 insertions(+), 4 deletions(-) create mode 100644 integrations/model-training/transformers/transformers-google-bert-fine-tuning/README.md diff --git a/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb index 514a6ab4..bff82cac 100644 --- a/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb +++ b/integrations/model-training/ray-train/notebooks/Comet_with_ray_train_huggingface_transformers.ipynb @@ -41,7 +41,7 @@ }, "outputs": [], "source": [ - "%pip install comet_ml \"ray[air]>=2.1.0\" \"transformers\" \"accelerate>=0.12.0\" \"datasets\" \"sentencepiece\" scipy \"scikit-learn\" protobuf \"torch>=1.3\" evaluate" + "%pip install \"comet_ml>=3.31.5\" \"ray[air]>=2.1.0\" \"transformers>=4.43.0\" \"accelerate>=0.12.0\" \"datasets\" \"sentencepiece\" scipy \"scikit-learn\" protobuf \"torch>=1.3\" evaluate" ] }, { diff --git a/integrations/model-training/transformers/transformers-distilbert-fine-tuning/README.md b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/README.md index 6975cc89..977b260b 100644 --- a/integrations/model-training/transformers/transformers-distilbert-fine-tuning/README.md +++ b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/README.md @@ -12,7 +12,7 @@ Instrument Transformers with Comet to start managing experiments, create dataset ## Documentation -For more information on using and configuring the Transformers integration, see: [https://www.comet.com/docs/v2/integrations/ml-frameworks/huggingface/](https://www.comet.com/docs/v2/integrations/ml-frameworks/huggingface/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=huggingface) +For more information on using and configuring the Transformers integration, see: [https://www.comet.com/docs/v2/integrations/ml-frameworks/huggingface/](https://www.comet.com/docs/v2/integrations/ml-frameworks/transformers/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=huggingface) ## See it diff --git a/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt index d9847054..c36869b0 100644 --- a/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt +++ b/integrations/model-training/transformers/transformers-distilbert-fine-tuning/requirements.txt @@ -3,4 +3,4 @@ comet_ml>=3.44.0 pandas scikit-learn torch -transformers>=4.42.4 +transformers>=4.43.0 diff --git a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/README.md b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/README.md new file mode 100644 index 00000000..afc9731b --- /dev/null +++ b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/README.md @@ -0,0 +1,37 @@ +# Transformers integration with Comet.ml + +[Hugging Face Transformers](https://github.com/huggingface/transformers) provide +general-purpose Machine Learning models for Natural Language +Understanding (NLP). Transformers give you easy access to pre-trained model +weights, and interoperability between PyTorch and TensorFlow. + +Instrument Transformers with Comet to start managing experiments, create dataset versions and track hyperparameters for faster and easier reproducibility and collaboration. + +Instrument Transformers with Comet to start managing experiments, create dataset versions and track hyperparameters for faster and easier reproducibility and collaboration. + + +## Documentation + +For more information on using and configuring the Transformers integration, see: [https://www.comet.com/docs/v2/integrations/ml-frameworks/huggingface/](https://www.comet.com/docs/v2/integrations/ml-frameworks/transformers/?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=huggingface) + +## See it + +Take a look at this [public Comet Project](https://www.comet.com/examples/comet-example-transformers-google-bert-fine-tuning/25d673e1153047eda82096f74142e2d0?utm_source=comet-examples&utm_medium=referral&utm_campaign=github_repo_2023&utm_content=pytorch). + +## Setup + +Install dependencies + +```bash +python -m pip install -r requirements.txt +``` + +## Run the example + + +This example shows how to use Comet in a HuggingFace Transformers script. + + +```bash +python transformers-distilbert-fine-tuning.py +``` diff --git a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt index 722db500..70ddd360 100644 --- a/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt +++ b/integrations/model-training/transformers/transformers-google-bert-fine-tuning/requirements.txt @@ -4,4 +4,4 @@ evaluate numpy scikit-learn torch -transformers[torch] +transformers[torch]>=4.43.0