video-captioning-exp.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bc0aed6b-1e2a-444c-b24d-3aebec2edf32",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hello World!\n"
     ]
    }
   ],
   "source": [
    "print(\"Hello World!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "70d6a2ed-3b81-4357-abb6-bdeae259e1b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8fe7eeda-aaa2-4c49-a786-070b8864ec6d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The file size of 'train_val_annotation/train_val_videodatainfo.json' is approximately 15.40 megabytes.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Specify the path to your JSON file\n",
    "json_file_path = \"train_val_annotation/train_val_videodatainfo.json\"\n",
    "\n",
    "try:\n",
    "    # Get the file size in bytes\n",
    "    file_size_bytes = os.path.getsize(json_file_path)\n",
    "\n",
    "    # Convert bytes to a more human-readable format (e.g., megabytes)\n",
    "    file_size_megabytes = file_size_bytes / (1024 * 1024)  # 1 MB = 1024 KB, 1 KB = 1024 bytes\n",
    "\n",
    "    print(f\"The file size of '{json_file_path}' is approximately {file_size_megabytes:.2f} megabytes.\")\n",
    "except FileNotFoundError:\n",
    "    print(f\"The file {json_file_path} was not found.\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ffa5240-c0a9-46fd-8557-64683f489f0b",
   "metadata": {},
   "source": [
    "# Main Code starts here"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c72d22e-b38c-4f0f-90e3-a4a443461f87",
   "metadata": {},
   "source": [
    "# Read the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ecf35cf5-3f35-434a-9101-2327ad6909a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    with open(json_file_path, \"r\", encoding=\"utf-8\") as json_file:\n",
    "        json_data = json.load(json_file)\n",
    "except FileNotFoundError:\n",
    "    print(f\"The file {json_file_path} was not found.\")\n",
    "except json.JSONDecodeError as e:\n",
    "    print(f\"Error decoding JSON: {e}\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6b0b39b6-e25e-4d90-9690-6d541ec89b1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['info', 'videos', 'sentences'])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fd58e70f-1c56-4b95-a346-5e7cf252e75c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'contributor': 'Microsoft MSM group',\n",
       " 'data_created': '2016-04-14 14:30:20',\n",
       " 'version': '1.0',\n",
       " 'description': 'This is 1.0 version of the 2016 MSR-VTT dataset.',\n",
       " 'year': '2016'}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_data[\"info\"] # useless"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "36d445d4-2818-448b-8f81-227a831b66f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'category': 9,\n",
       "  'url': 'https://www.youtube.com/watch?v=9lZi22qLlEo',\n",
       "  'video_id': 'video0',\n",
       "  'start time': 137.72,\n",
       "  'end time': 149.44,\n",
       "  'split': 'train',\n",
       "  'id': 0},\n",
       " {'category': 16,\n",
       "  'url': 'https://www.youtube.com/watch?v=w4JM08PDEng',\n",
       "  'video_id': 'video1',\n",
       "  'start time': 184.33,\n",
       "  'end time': 206.89,\n",
       "  'split': 'train',\n",
       "  'id': 1},\n",
       " {'category': 9,\n",
       "  'url': 'https://www.youtube.com/watch?v=QA7KVQq9vKA',\n",
       "  'video_id': 'video2',\n",
       "  'start time': 31.17,\n",
       "  'end time': 41.24,\n",
       "  'split': 'train',\n",
       "  'id': 2},\n",
       " {'category': 8,\n",
       "  'url': 'https://www.youtube.com/watch?v=QFmJZ0GU6yc',\n",
       "  'video_id': 'video3',\n",
       "  'start time': 48.26,\n",
       "  'end time': 58.51,\n",
       "  'split': 'train',\n",
       "  'id': 3},\n",
       " {'category': 14,\n",
       "  'url': 'https://www.youtube.com/watch?v=2q-dONPhzis',\n",
       "  'video_id': 'video4',\n",
       "  'start time': 268.58,\n",
       "  'end time': 278.83,\n",
       "  'split': 'train',\n",
       "  'id': 4}]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_data[\"videos\"][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4dc4ed09-2e13-427e-818d-3a4455d32221",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'caption': 'a cartoon animals runs through an ice cave in a video game',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 0},\n",
       " {'caption': 'a cartoon character runs around inside of a video game',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 1},\n",
       " {'caption': 'a character is running in the snow',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 2},\n",
       " {'caption': 'a person plays a video game centered around ice age the movie',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 3},\n",
       " {'caption': 'a person plays online and records themselves',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 4}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_data[\"sentences\"][:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4397da86-3b94-4df3-8ca8-41b7046dd14a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(140200, 7010)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(json_data[\"sentences\"]), len(json_data[\"videos\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "01c753f0-b221-47ab-8291-67ada5372d3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "int(140200/7010) # 20 captions per video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1d159cdb-efbb-4199-bf35-d39f9778e1b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_json_data = {\n",
    "    \"info\": json_data[\"info\"],\n",
    "    \"videos\": json_data[\"videos\"][:5],\n",
    "    \"sentences\": json_data[\"sentences\"][:5]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "34727ee3-237a-41f2-9e24-0f46b2b8349b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'info': {'contributor': 'Microsoft MSM group',\n",
       "  'data_created': '2016-04-14 14:30:20',\n",
       "  'version': '1.0',\n",
       "  'description': 'This is 1.0 version of the 2016 MSR-VTT dataset.',\n",
       "  'year': '2016'},\n",
       " 'videos': [{'category': 9,\n",
       "   'url': 'https://www.youtube.com/watch?v=9lZi22qLlEo',\n",
       "   'video_id': 'video0',\n",
       "   'start time': 137.72,\n",
       "   'end time': 149.44,\n",
       "   'split': 'train',\n",
       "   'id': 0},\n",
       "  {'category': 16,\n",
       "   'url': 'https://www.youtube.com/watch?v=w4JM08PDEng',\n",
       "   'video_id': 'video1',\n",
       "   'start time': 184.33,\n",
       "   'end time': 206.89,\n",
       "   'split': 'train',\n",
       "   'id': 1},\n",
       "  {'category': 9,\n",
       "   'url': 'https://www.youtube.com/watch?v=QA7KVQq9vKA',\n",
       "   'video_id': 'video2',\n",
       "   'start time': 31.17,\n",
       "   'end time': 41.24,\n",
       "   'split': 'train',\n",
       "   'id': 2},\n",
       "  {'category': 8,\n",
       "   'url': 'https://www.youtube.com/watch?v=QFmJZ0GU6yc',\n",
       "   'video_id': 'video3',\n",
       "   'start time': 48.26,\n",
       "   'end time': 58.51,\n",
       "   'split': 'train',\n",
       "   'id': 3},\n",
       "  {'category': 14,\n",
       "   'url': 'https://www.youtube.com/watch?v=2q-dONPhzis',\n",
       "   'video_id': 'video4',\n",
       "   'start time': 268.58,\n",
       "   'end time': 278.83,\n",
       "   'split': 'train',\n",
       "   'id': 4}],\n",
       " 'sentences': [{'caption': 'a cartoon animals runs through an ice cave in a video game',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 0},\n",
       "  {'caption': 'a cartoon character runs around inside of a video game',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 1},\n",
       "  {'caption': 'a character is running in the snow',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 2},\n",
       "  {'caption': 'a person plays a video game centered around ice age the movie',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 3},\n",
       "  {'caption': 'a person plays online and records themselves',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 4}]}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_json_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6eaacbda-eb0a-4b25-84b4-8fb0c70ad0fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'category': 9,\n",
       "  'url': 'https://www.youtube.com/watch?v=9lZi22qLlEo',\n",
       "  'video_id': 'video0',\n",
       "  'start time': 137.72,\n",
       "  'end time': 149.44,\n",
       "  'split': 'train',\n",
       "  'id': 0},\n",
       " {'category': 16,\n",
       "  'url': 'https://www.youtube.com/watch?v=w4JM08PDEng',\n",
       "  'video_id': 'video1',\n",
       "  'start time': 184.33,\n",
       "  'end time': 206.89,\n",
       "  'split': 'train',\n",
       "  'id': 1},\n",
       " {'category': 9,\n",
       "  'url': 'https://www.youtube.com/watch?v=QA7KVQq9vKA',\n",
       "  'video_id': 'video2',\n",
       "  'start time': 31.17,\n",
       "  'end time': 41.24,\n",
       "  'split': 'train',\n",
       "  'id': 2},\n",
       " {'category': 8,\n",
       "  'url': 'https://www.youtube.com/watch?v=QFmJZ0GU6yc',\n",
       "  'video_id': 'video3',\n",
       "  'start time': 48.26,\n",
       "  'end time': 58.51,\n",
       "  'split': 'train',\n",
       "  'id': 3},\n",
       " {'category': 14,\n",
       "  'url': 'https://www.youtube.com/watch?v=2q-dONPhzis',\n",
       "  'video_id': 'video4',\n",
       "  'start time': 268.58,\n",
       "  'end time': 278.83,\n",
       "  'split': 'train',\n",
       "  'id': 4}]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_json_data[\"videos\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "9e039f8b-f751-4afb-ba83-f5fc62dd9c10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'caption': 'a cartoon animals runs through an ice cave in a video game',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 0},\n",
       " {'caption': 'a cartoon character runs around inside of a video game',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 1},\n",
       " {'caption': 'a character is running in the snow',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 2},\n",
       " {'caption': 'a person plays a video game centered around ice age the movie',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 3},\n",
       " {'caption': 'a person plays online and records themselves',\n",
       "  'video_id': 'video2960',\n",
       "  'sen_id': 4}]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_json_data[\"sentences\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "110c8216-6a5a-4239-9a38-7389f875965b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['video5351.mp4',\n",
       " 'video1149.mp4',\n",
       " 'video5207.mp4',\n",
       " 'video20.mp4',\n",
       " 'video12.mp4']"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(\"TrainValVideo\")[:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "534f3bc1-2f57-46ad-b690-15940a7c116b",
   "metadata": {},
   "source": [
    "# Preprocessing Video Frames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "2513b7ae-7b93-42b0-9447-ac6a79c81111",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pip in ./venv/lib/python3.9/site-packages (23.2.1)\n",
      "Requirement already satisfied: opencv-python in ./venv/lib/python3.9/site-packages (4.8.0.76)\n",
      "Requirement already satisfied: numpy>=1.17.0 in ./venv/lib/python3.9/site-packages (from opencv-python) (1.25.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install --upgrade pip\n",
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "e3995908-4288-44fe-8774-8504818bd8a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7c91e4df-f75c-4d41-a100-1bf5154e9922",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_frame(frame):\n",
    "    # Resize frame to a fixed size (e.g., 224x224)\n",
    "    frame = cv2.resize(frame, (224, 224))\n",
    "    # Normalize pixel values to [0, 1] and convert to PyTorch tensor\n",
    "    frame = torch.tensor(frame / 255.0, dtype=torch.float32)\n",
    "    return frame\n",
    "\n",
    "# Load a video and preprocess its frames\n",
    "cap = cv2.VideoCapture(\"TrainValVideo/video0.mp4\")\n",
    "frames = []\n",
    "\n",
    "while True:\n",
    "    ret, frame = cap.read()\n",
    "    if not ret:\n",
    "        break\n",
    "    frame = preprocess_frame(frame)\n",
    "    frames.append(frame)\n",
    "\n",
    "cap.release()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b25a5c3-e70c-4f83-af1d-79d0243a9905",
   "metadata": {},
   "source": [
    "Data Preparation:\n",
    "\n",
    "Download the Dataset: First, download the MSR-VTT dataset, which includes video files and associated captions.\n",
    "\n",
    "Preprocess Video Frames: Extract video frames from the video files. Depending on your chosen video feature extraction approach, you may need to sample frames at regular intervals, extract optical flow, or use other techniques to preprocess the videos.\n",
    "\n",
    "Tokenize Captions: Tokenize the captions into subword tokens using a suitable tokenizer. BERT models typically work with subword tokens, and you can use libraries like Hugging Face Transformers for tokenization.\n",
    "\n",
    "Create Data Splits: Divide the dataset into training, validation, and test sets.\n",
    "\n",
    "Feature Extraction:\n",
    "\n",
    "Video Feature Extraction: Extract video features from the preprocessed video frames using a Vision Transformer (ViT) model. You can use pre-trained ViT models for this purpose. Some libraries like Hugging Face Transformers may provide pre-trained ViT models that can be fine-tuned for feature extraction on your specific dataset.\n",
    "\n",
    "Text Embeddings: Convert the tokenized captions into BERT embeddings. You can use a pre-trained BERT model to encode the captions into dense representations.\n",
    "\n",
    "Model Architecture:\n",
    "\n",
    "Combine Vision and Text: Create a model architecture that combines the extracted video features and BERT embeddings. This can be done through various fusion techniques, such as concatenation, attention mechanisms, or multimodal embeddings.\n",
    "\n",
    "Caption Generation: Design a caption generation model (e.g., recurrent neural network or transformer-based) that takes the fused features as input and generates captions one token at a time.\n",
    "\n",
    "Training:\n",
    "\n",
    "Loss Function: Define an appropriate loss function for training the caption generation model. Common choices include cross-entropy loss or custom loss functions that encourage diversity and fluency in generated captions.\n",
    "\n",
    "Training Procedure: Train the model using the training split of the dataset. Monitor performance on the validation set to prevent overfitting.\n",
    "\n",
    "Inference:\n",
    "\n",
    "Beam Search or Sampling: During inference, use the trained model to generate captions for test videos. You can use techniques like beam search or sampling to generate diverse captions for each video.\n",
    "Evaluation:\n",
    "\n",
    "Use standard evaluation metrics like BLEU, METEOR, ROUGE, and CIDEr to assess the quality of generated captions. Evaluate the model on the test set to measure its performance.\n",
    "Fine-Tuning:\n",
    "\n",
    "Depending on the results, you may consider fine-tuning the model's hyperparameters or architecture to improve caption quality.\n",
    "Deployment:\n",
    "\n",
    "Once you are satisfied with the model's performance, you can deploy it for generating captions for new videos."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "68e368ca-f70b-49c2-81ea-b780da2455c9",
   "metadata": {},
   "source": [
    "# Batch Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "22ed8fba-fc9e-4327-b73a-b846a060b2e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([224, 224, 3])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frames[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "47d6ce39-bb21-4d97-ba8f-b0bfd763182b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "# Assume frames is a list of preprocessed video frames\n",
    "frames_tensor = torch.stack(frames)  # Convert the list to a tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cad2dca6-2d10-4fd6-a3bb-cda17bf5e558",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/abhijoysarkar/video-captioning/venv/lib/python3.9/site-packages/transformers/models/vit/feature_extraction_vit.py:28: FutureWarning: The class ViTFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please use ViTImageProcessor instead.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "ViTModel(\n",
       "  (embeddings): ViTEmbeddings(\n",
       "    (patch_embeddings): ViTPatchEmbeddings(\n",
       "      (projection): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "    )\n",
       "    (dropout): Dropout(p=0.0, inplace=False)\n",
       "  )\n",
       "  (encoder): ViTEncoder(\n",
       "    (layer): ModuleList(\n",
       "      (0-11): 12 x ViTLayer(\n",
       "        (attention): ViTAttention(\n",
       "          (attention): ViTSelfAttention(\n",
       "            (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "          (output): ViTSelfOutput(\n",
       "            (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "            (dropout): Dropout(p=0.0, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (intermediate): ViTIntermediate(\n",
       "          (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          (intermediate_act_fn): GELUActivation()\n",
       "        )\n",
       "        (output): ViTOutput(\n",
       "          (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "          (dropout): Dropout(p=0.0, inplace=False)\n",
       "        )\n",
       "        (layernorm_before): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "        (layernorm_after): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (layernorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "  (pooler): ViTPooler(\n",
       "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "    (activation): Tanh()\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from transformers import ViTFeatureExtractor, ViTModel\n",
    "import torch\n",
    "\n",
    "# Load the ViT feature extractor and model\n",
    "feature_extractor = ViTFeatureExtractor.from_pretrained(\"google/vit-base-patch16-224-in21k\")\n",
    "model = ViTModel.from_pretrained(\"google/vit-base-patch16-224-in21k\")\n",
    "model.eval()  # Set the model to evaluation mode"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4955d47f-7f84-485e-8d15-784629579655",
   "metadata": {},
   "source": [
    "# Extract video features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "5e49fbff-22a4-4850-95e4-c6cc7dc8ea90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU 0 - GPU Name: Tesla T4\n",
      "  GPU Usage: 0.0%\n",
      "  GPU Memory Usage: 10463.0 MB out of 15360.0 MB\n"
     ]
    }
   ],
   "source": [
    "import GPUtil\n",
    "\n",
    "def monitor_gpu():\n",
    "    GPUs = GPUtil.getGPUs()\n",
    "    for i, gpu in enumerate(GPUs):\n",
    "        print(f\"GPU {i} - GPU Name: {gpu.name}\")\n",
    "        print(f\"  GPU Usage: {gpu.load * 100}%\")\n",
    "        print(f\"  GPU Memory Usage: {gpu.memoryFree} MB out of {gpu.memoryTotal} MB\")\n",
    "\n",
    "monitor_gpu()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0519fafd-300a-44d6-a738-21ada26b6eb4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming frames_tensor is your tensor of preprocessed frames\n",
    "with torch.no_grad():\n",
    "    # Use the feature extractor to prepare the input\n",
    "    inputs = feature_extractor(images=frames_tensor, return_tensors=\"pt\")\n",
    "\n",
    "    # Move the entire inputs dictionary to GPU\n",
    "    inputs = {key: value.to(\"cuda\") for key, value in inputs.items()}\n",
    "    model = model.to(\"cuda\")\n",
    "    \n",
    "    # Forward pass through the model to get the features\n",
    "    outputs = model(**inputs)\n",
    "    \n",
    "    # Extract the features from the model's output\n",
    "    features = outputs.last_hidden_state  # This contains the features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a96333e3-18c5-41fa-b504-2ff034df8e1e",
   "metadata": {},
   "source": [
    "# BERT Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8293555d-57e4-4b67-b0f7-aab995ff4af2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'info': {'contributor': 'Microsoft MSM group',\n",
       "  'data_created': '2016-04-14 14:30:20',\n",
       "  'version': '1.0',\n",
       "  'description': 'This is 1.0 version of the 2016 MSR-VTT dataset.',\n",
       "  'year': '2016'},\n",
       " 'videos': [{'category': 9,\n",
       "   'url': 'https://www.youtube.com/watch?v=9lZi22qLlEo',\n",
       "   'video_id': 'video0',\n",
       "   'start time': 137.72,\n",
       "   'end time': 149.44,\n",
       "   'split': 'train',\n",
       "   'id': 0},\n",
       "  {'category': 16,\n",
       "   'url': 'https://www.youtube.com/watch?v=w4JM08PDEng',\n",
       "   'video_id': 'video1',\n",
       "   'start time': 184.33,\n",
       "   'end time': 206.89,\n",
       "   'split': 'train',\n",
       "   'id': 1},\n",
       "  {'category': 9,\n",
       "   'url': 'https://www.youtube.com/watch?v=QA7KVQq9vKA',\n",
       "   'video_id': 'video2',\n",
       "   'start time': 31.17,\n",
       "   'end time': 41.24,\n",
       "   'split': 'train',\n",
       "   'id': 2},\n",
       "  {'category': 8,\n",
       "   'url': 'https://www.youtube.com/watch?v=QFmJZ0GU6yc',\n",
       "   'video_id': 'video3',\n",
       "   'start time': 48.26,\n",
       "   'end time': 58.51,\n",
       "   'split': 'train',\n",
       "   'id': 3},\n",
       "  {'category': 14,\n",
       "   'url': 'https://www.youtube.com/watch?v=2q-dONPhzis',\n",
       "   'video_id': 'video4',\n",
       "   'start time': 268.58,\n",
       "   'end time': 278.83,\n",
       "   'split': 'train',\n",
       "   'id': 4}],\n",
       " 'sentences': [{'caption': 'a cartoon animals runs through an ice cave in a video game',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 0},\n",
       "  {'caption': 'a cartoon character runs around inside of a video game',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 1},\n",
       "  {'caption': 'a character is running in the snow',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 2},\n",
       "  {'caption': 'a person plays a video game centered around ice age the movie',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 3},\n",
       "  {'caption': 'a person plays online and records themselves',\n",
       "   'video_id': 'video2960',\n",
       "   'sen_id': 4}]}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_json_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7d50ecb3-22d4-430a-bc89-b25e21529315",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Video ID: video0\n",
      "Category: 9\n",
      "URL: https://www.youtube.com/watch?v=9lZi22qLlEo\n",
      "Start Time: 137.72\n",
      "End Time: 149.44\n",
      "Split: train\n",
      "ID: 0\n"
     ]
    }
   ],
   "source": [
    "# Assuming json_data is your JSON data\n",
    "video_id_to_extract = 'video0'\n",
    "\n",
    "# Find the dictionary for \"video0\" in the \"videos\" list\n",
    "video_info = next((video for video in json_data['videos'] if video['video_id'] == video_id_to_extract), None)\n",
    "\n",
    "if video_info:\n",
    "    # Print or use the extracted information\n",
    "    print(\"Video ID:\", video_info['video_id'])\n",
    "    print(\"Category:\", video_info['category'])\n",
    "    print(\"URL:\", video_info['url'])\n",
    "    print(\"Start Time:\", video_info['start time'])\n",
    "    print(\"End Time:\", video_info['end time'])\n",
    "    print(\"Split:\", video_info['split'])\n",
    "    print(\"ID:\", video_info['id'])\n",
    "else:\n",
    "    print(f\"Video with ID {video_id_to_extract} not found.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "95b7fc84-420b-4488-b2e2-dd819ca49468",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Captions for Video0:\n",
      "a car is shown\n",
      "a group is dancing\n",
      "a man drives a vehicle through the countryside\n",
      "a man drives down the road in an audi\n",
      "a man driving a car\n",
      "a man is driving a car\n",
      "a man is driving down a road\n",
      "a man is driving in a car as part of a commercial\n",
      "a man is driving\n",
      "a man riding the car speedly in a narrow road\n",
      "a man showing the various features of a car\n",
      "a man silently narrates his experience driving an audi\n",
      "a person is driving his car around curves in the road\n",
      "a person telling about a car\n",
      "guy driving a car down the road\n",
      "man talking about a car while driving\n",
      "the man drives the car\n",
      "the man driving the audi as smooth as possible\n",
      "a man is driving\n",
      "guy driving a car down the road\n"
     ]
    }
   ],
   "source": [
    "# Find all captions for \"video0\" in the \"sentences\" list\n",
    "captions_for_video0 = [sentence['caption'] for sentence in json_data['sentences'] if sentence['video_id'] == video_id_to_extract]\n",
    "\n",
    "# Print or use the extracted captions\n",
    "if captions_for_video0:\n",
    "    print(\"Captions for Video0:\")\n",
    "    for caption in captions_for_video0:\n",
    "        print(caption)\n",
    "else:\n",
    "    print(f\"No captions found for Video ID {video_id_to_extract}.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "4d50ab20-0142-4ab3-9830-fd00f4294ea8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import BertTokenizer, BertModel\n",
    "import torch\n",
    "\n",
    "# Load the BERT tokenizer and model\n",
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "model = BertModel.from_pretrained(\"bert-base-uncased\")\n",
    "\n",
    "# TODO: Change this line later for being more generic\n",
    "captions = captions_for_video0\n",
    "\n",
    "# Use batch_encode_plus to tokenize and pad the captions as a batch\n",
    "encoding = tokenizer.batch_encode_plus(\n",
    "    captions,\n",
    "    return_tensors=\"pt\",\n",
    "    padding=True,  # Pad to the maximum sequence length within the batch\n",
    "    truncation=True,  # Truncate to the maximum sequence length if exceeded\n",
    "    max_length=128,  # Set the maximum sequence length\n",
    "    return_attention_mask=True,\n",
    ")\n",
    "\n",
    "# Forward pass through the BERT model to get the embeddings\n",
    "with torch.no_grad():\n",
    "    outputs = model(**encoding)\n",
    "\n",
    "# Extract the embeddings from the BERT model's output\n",
    "embeddings = outputs.last_hidden_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "3e4a80e3-fa1f-4b24-89d3-6a6d9f049ccb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([20, 14, 768])"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c9be1240-2fdf-4555-bc6c-780b8a80494c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([300, 197, 768])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "07778dee-9024-4b6b-ad19-29ca8c6f2d5a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([300, 224, 224, 3])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "frames_tensor.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b37996ae-2b7d-4d89-8f02-512e77a0abbd",
   "metadata": {},
   "source": [
    "Encoding Modalities:\n",
    "\n",
    "Encode the text (captions) using a language model like BERT to obtain text embeddings.\n",
    "Extract video features using a Vision Transformer (ViT) or similar model to obtain video embeddings.\n",
    "Attention Mechanism:\n",
    "\n",
    "Use an attention mechanism (e.g., self-attention or cross-modal attention) to compute attention scores that capture the relationships between the text and video embeddings.\n",
    "Combine the text and video embeddings based on these attention scores. This step typically involves weighted summation or concatenation.\n",
    "Decoding and Caption Generation:\n",
    "\n",
    "Feed the combined embeddings into a captioning model (e.g., an LSTM or transformer-based decoder) to generate captions."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bbb58778-8d52-443d-a28d-ce3f48de1f77",
   "metadata": {},
   "source": [
    "For the MSR-VTT dataset specifically, you might consider starting with an architecture like \"Show, Attend, and Tell (SAT),\" which combines visual attention with text decoding. This architecture has been successful for video captioning tasks and uses an attention mechanism to focus on relevant video regions while generating captions.\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2bcc32e-5908-4263-b6a6-ce3098c2664a",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# SAT Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "98ff6827-b1b8-47c2-af2d-73ef650b9a1c",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index out of range in self",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[38], line 89\u001b[0m\n\u001b[1;32m     86\u001b[0m video_features \u001b[38;5;241m=\u001b[39m features  \u001b[38;5;66;03m# Assuming you have the video features\u001b[39;00m\n\u001b[1;32m     87\u001b[0m captions \u001b[38;5;241m=\u001b[39m captions_indices  \u001b[38;5;66;03m# Use the Long tensor for indices\u001b[39;00m\n\u001b[0;32m---> 89\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[43mmodel\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvideo_features\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcaptions\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     91\u001b[0m \u001b[38;5;66;03m# Compute loss and backpropagate\u001b[39;00m\n\u001b[1;32m     92\u001b[0m loss \u001b[38;5;241m=\u001b[39m criterion(outputs\u001b[38;5;241m.\u001b[39mview(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m, num_words), captions\u001b[38;5;241m.\u001b[39mview(\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m))\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "Cell \u001b[0;32mIn[38], line 43\u001b[0m, in \u001b[0;36mSATModel.forward\u001b[0;34m(self, video_features, captions)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[38;5;66;03m# Loop through time steps\u001b[39;00m\n\u001b[1;32m     41\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m t \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(max_seq_length):\n\u001b[1;32m     42\u001b[0m     \u001b[38;5;66;03m# LSTM input: previous word embedding and context vector\u001b[39;00m\n\u001b[0;32m---> 43\u001b[0m     word_embed \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mword_embeddings\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcaptions\u001b[49m\u001b[43m[\u001b[49m\u001b[43m:\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mt\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m:\u001b[49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     44\u001b[0m     lstm_input \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mcat((word_embed, h), dim\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n\u001b[1;32m     46\u001b[0m     \u001b[38;5;66;03m# LSTM step\u001b[39;00m\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/modules/sparse.py:162\u001b[0m, in \u001b[0;36mEmbedding.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 162\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    163\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_norm\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    164\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnorm_type\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale_grad_by_freq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msparse\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/functional.py:2210\u001b[0m, in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m   2204\u001b[0m     \u001b[38;5;66;03m# Note [embedding_renorm set_grad_enabled]\u001b[39;00m\n\u001b[1;32m   2205\u001b[0m     \u001b[38;5;66;03m# XXX: equivalent to\u001b[39;00m\n\u001b[1;32m   2206\u001b[0m     \u001b[38;5;66;03m# with torch.no_grad():\u001b[39;00m\n\u001b[1;32m   2207\u001b[0m     \u001b[38;5;66;03m#   torch.embedding_renorm_\u001b[39;00m\n\u001b[1;32m   2208\u001b[0m     \u001b[38;5;66;03m# remove once script supports set_grad_enabled\u001b[39;00m\n\u001b[1;32m   2209\u001b[0m     _no_grad_embedding_renorm_(weight, \u001b[38;5;28minput\u001b[39m, max_norm, norm_type)\n\u001b[0;32m-> 2210\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpadding_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscale_grad_by_freq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msparse\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mIndexError\u001b[0m: index out of range in self"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "class SATModel(nn.Module):\n",
    "    def __init__(self, video_feature_dim, hidden_dim, num_words):\n",
    "        super(SATModel, self).__init__()\n",
    "\n",
    "        # Video Feature Module (e.g., Vision Transformer)\n",
    "        self.video_feature_dim = video_feature_dim\n",
    "        self.video_feature_extractor = nn.Sequential(\n",
    "            nn.Linear(video_feature_dim, hidden_dim),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(hidden_dim, hidden_dim)\n",
    "        )\n",
    "\n",
    "        # LSTM Decoder with Attention\n",
    "        self.lstm = nn.LSTMCell(hidden_dim, hidden_dim)\n",
    "        self.attention_linear = nn.Linear(hidden_dim, self.video_feature_dim)  # Automatically determine the video feature dimension\n",
    "        self.context_linear = nn.Linear(self.video_feature_dim, hidden_dim)\n",
    "\n",
    "        # Caption Generation Module\n",
    "        self.caption_linear = nn.Linear(hidden_dim, num_words)\n",
    "\n",
    "    def forward(self, video_features, captions):\n",
    "        batch_size, max_seq_length, _ = captions.size()\n",
    "        _, num_video_frames, _ = video_features.size()  # Automatically determine the number of video frames\n",
    "\n",
    "        # Initialize hidden states\n",
    "        h = torch.zeros(batch_size, self.lstm.hidden_size).to(captions.device)\n",
    "        c = torch.zeros(batch_size, self.lstm.hidden_size).to(captions.device)\n",
    "\n",
    "        # Lists to store generated captions\n",
    "        generated_captions = []\n",
    "\n",
    "        # Loop through time steps\n",
    "        for t in range(max_seq_length):\n",
    "            # LSTM input: context vector\n",
    "            lstm_input = h\n",
    "\n",
    "            # LSTM step\n",
    "            h, c = self.lstm(lstm_input, (h, c))\n",
    "\n",
    "            # Compute attention scores\n",
    "            attention_scores = F.softmax(self.attention_linear(h), dim=1)\n",
    "\n",
    "            # Calculate context vector\n",
    "            context_vector = torch.bmm(attention_scores.unsqueeze(1), video_features)\n",
    "            context_vector = self.context_linear(context_vector.squeeze(1))\n",
    "\n",
    "            # Combine LSTM output and context vector\n",
    "            combined_output = h + context_vector\n",
    "\n",
    "            # Generate captions\n",
    "            caption_scores = self.caption_linear(combined_output)\n",
    "            generated_captions.append(caption_scores)\n",
    "\n",
    "        # Stack caption scores into a tensor\n",
    "        generated_captions = torch.stack(generated_captions, dim=1)\n",
    "\n",
    "        return generated_captions\n",
    "\n",
    "# Initialize the SAT model\n",
    "video_feature_dim = features.size(0)  # Automatically determine the video feature dimension\n",
    "hidden_dim = 512  # Adjust as needed\n",
    "num_words = 10000  # Adjust based on your vocabulary size\n",
    "\n",
    "model = SATModel(video_feature_dim, hidden_dim, num_words)\n",
    "\n",
    "# Define your loss and optimizer for training\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# Forward pass with your video features and embeddings\n",
    "video_features = features  # Assuming you have the video features\n",
    "captions = embeddings  # Assuming you have the embeddings for captions\n",
    "\n",
    "outputs = model(video_features, captions)\n",
    "\n",
    "# Compute loss and backpropagate\n",
    "loss = criterion(outputs.view(-1, num_words), captions.view(-1))\n",
    "optimizer.zero_grad()\n",
    "loss.backward()\n",
    "optimizer.step()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6c5d3445-db36-4d45-8f65-89ad48290472",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([20, 14, 768])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "c1669a44-c744-41a1-a4d1-bbaa8dcabfe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([300, 197, 768])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "19e745f1-94fe-4474-ae38-aa7dc0e72e64",
   "metadata": {},
   "outputs": [],
   "source": [
    "embed = nn.Embedding(14, 768)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c458924-632b-4ad2-9256-d3c36feb1475",
   "metadata": {},
   "outputs": [],
   "source": [
    "for t in range(max_seq_length):\n",
    "            # LSTM input: previous word embedding and context vector\n",
    "            word_embed = self.word_embeddings(captions[:, t, :])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "dd900bcf-dc87-4611-8679-d2469fd41eb1",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index out of range in self",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[44], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43membed\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcaptions\u001b[49m\u001b[43m[\u001b[49m\u001b[38;5;241;43m0\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/modules/sparse.py:162\u001b[0m, in \u001b[0;36mEmbedding.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 162\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    163\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_norm\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    164\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnorm_type\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale_grad_by_freq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msparse\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/functional.py:2210\u001b[0m, in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m   2204\u001b[0m     \u001b[38;5;66;03m# Note [embedding_renorm set_grad_enabled]\u001b[39;00m\n\u001b[1;32m   2205\u001b[0m     \u001b[38;5;66;03m# XXX: equivalent to\u001b[39;00m\n\u001b[1;32m   2206\u001b[0m     \u001b[38;5;66;03m# with torch.no_grad():\u001b[39;00m\n\u001b[1;32m   2207\u001b[0m     \u001b[38;5;66;03m#   torch.embedding_renorm_\u001b[39;00m\n\u001b[1;32m   2208\u001b[0m     \u001b[38;5;66;03m# remove once script supports set_grad_enabled\u001b[39;00m\n\u001b[1;32m   2209\u001b[0m     _no_grad_embedding_renorm_(weight, \u001b[38;5;28minput\u001b[39m, max_norm, norm_type)\n\u001b[0;32m-> 2210\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpadding_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscale_grad_by_freq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msparse\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mIndexError\u001b[0m: index out of range in self"
     ]
    }
   ],
   "source": [
    "embed(captions[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "a68d5f28-c32e-4afd-8528-dd5f8262291b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assuming you have BERT embeddings of shape [20, 14, 768]\n",
    "bert_embeddings = torch.randn(20, 14, 768)\n",
    "\n",
    "# Define the vocabulary size and embedding dimension\n",
    "vocab_size = 1000  # Adjust this according to your use case\n",
    "embedding_dim = 300  # Adjust this according to your use case\n",
    "\n",
    "# Create an nn.Embedding layer\n",
    "embedding_layer = nn.Embedding(vocab_size, embedding_dim)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c24d8a84-4baa-46ab-89c8-527ebbf9a72f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.2598, -0.4100, -2.0121,  ..., -0.1620,  0.3682,  1.7746],\n",
       "        [ 1.0279, -0.7828, -0.5770,  ..., -0.2357,  0.6445,  0.4374],\n",
       "        [-0.3376,  1.3353,  0.9249,  ...,  1.5208,  1.4703,  2.5637],\n",
       "        ...,\n",
       "        [-2.3290,  0.5772,  0.6292,  ...,  0.3657,  0.0304,  0.6979],\n",
       "        [ 0.1277, -1.3416,  1.1254,  ...,  0.4654,  0.2355,  0.1187],\n",
       "        [-0.4762, -1.2175,  2.0629,  ...,  0.9348, -1.2710, -0.1460]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Assuming you want to \"embed\" the BERT embeddings\n",
    "# Reshape the BERT embeddings to match the expected shape for nn.Embedding\n",
    "bert_embeddings_flat = bert_embeddings.view(-1, 768)\n",
    "\n",
    "bert_embeddings_flat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "36b415e3-e604-49f9-a696-4ff90644230d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0,  0, -2,  ...,  0,  0,  1],\n",
       "        [ 1,  0,  0,  ...,  0,  0,  0],\n",
       "        [ 0,  1,  0,  ...,  1,  1,  2],\n",
       "        ...,\n",
       "        [-2,  0,  0,  ...,  0,  0,  0],\n",
       "        [ 0, -1,  1,  ...,  0,  0,  0],\n",
       "        [ 0, -1,  2,  ...,  0, -1,  0]])"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_embeddings_flat.long()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d442842b-c575-473d-be41-f9e90df57b6d",
   "metadata": {},
   "source": [
    "https://github.com/sgrvinod/a-PyTorch-Tutorial-to-Image-Captioning/tree/master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "a7466f4d-f761-47d0-afd8-6b11c96525e6",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "index out of range in self",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[50], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# Pass the reshaped BERT embeddings through the nn.Embedding layer\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m embedded_result \u001b[38;5;241m=\u001b[39m \u001b[43membedding_layer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mbert_embeddings_flat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlong\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;66;03m# Now, the embedded_result will be of shape [20 * 14, 768]\u001b[39;00m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;66;03m# You can reshape it back to [20, 14, 768] if needed\u001b[39;00m\n\u001b[1;32m      6\u001b[0m embedded_result \u001b[38;5;241m=\u001b[39m embedded_result\u001b[38;5;241m.\u001b[39mview(\u001b[38;5;241m20\u001b[39m, \u001b[38;5;241m14\u001b[39m, \u001b[38;5;241m768\u001b[39m)\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/modules/sparse.py:162\u001b[0m, in \u001b[0;36mEmbedding.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    161\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 162\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    163\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mpadding_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmax_norm\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    164\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mnorm_type\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mscale_grad_by_freq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msparse\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/video-captioning/venv/lib/python3.9/site-packages/torch/nn/functional.py:2210\u001b[0m, in \u001b[0;36membedding\u001b[0;34m(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)\u001b[0m\n\u001b[1;32m   2204\u001b[0m     \u001b[38;5;66;03m# Note [embedding_renorm set_grad_enabled]\u001b[39;00m\n\u001b[1;32m   2205\u001b[0m     \u001b[38;5;66;03m# XXX: equivalent to\u001b[39;00m\n\u001b[1;32m   2206\u001b[0m     \u001b[38;5;66;03m# with torch.no_grad():\u001b[39;00m\n\u001b[1;32m   2207\u001b[0m     \u001b[38;5;66;03m#   torch.embedding_renorm_\u001b[39;00m\n\u001b[1;32m   2208\u001b[0m     \u001b[38;5;66;03m# remove once script supports set_grad_enabled\u001b[39;00m\n\u001b[1;32m   2209\u001b[0m     _no_grad_embedding_renorm_(weight, \u001b[38;5;28minput\u001b[39m, max_norm, norm_type)\n\u001b[0;32m-> 2210\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mtorch\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mpadding_idx\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mscale_grad_by_freq\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43msparse\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mIndexError\u001b[0m: index out of range in self"
     ]
    }
   ],
   "source": [
    "# Pass the reshaped BERT embeddings through the nn.Embedding layer\n",
    "embedded_result = embedding_layer(bert_embeddings_flat.long())\n",
    "\n",
    "# Now, the embedded_result will be of shape [20 * 14, 768]\n",
    "# You can reshape it back to [20, 14, 768] if needed\n",
    "embedded_result = embedded_result.view(20, 14, 768)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "037df103-dc47-46e8-9255-0f35985a6946",
   "metadata": {},
   "source": [
    ">Caption Embedding at input is so useless\n",
    "\n",
    "https://github.com/sgrvinod/a-PyTorch-Tutorial-to-Image-Captioning/tree/master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2f005829-dcef-44f8-9f71-fe61ae6b8ef0",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'bert_embeddings_flat' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mbert_embeddings_flat\u001b[49m\u001b[38;5;241m.\u001b[39mshape\n",
      "\u001b[0;31mNameError\u001b[0m: name 'bert_embeddings_flat' is not defined"
     ]
    }
   ],
   "source": [
    "bert_embeddings_flat.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "41a4b4b5-3912-4531-9744-873100c45903",
   "metadata": {},
   "source": [
    "# Working Model Code here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "404a11e3-8e59-4685-b8ae-4ef0836dec79",
   "metadata": {},
   "outputs": [],
   "source": [
    "pooled_embeddings = torch.mean(embeddings, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "507ade88-f185-46d7-baab-ac1d00c0a067",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([14, 768])"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pooled_embeddings.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "d9767593-2bcd-43fa-85a4-277164ffddaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "pooled_features = torch.mean(features, dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "9121ff7e-9e5f-4e6c-9724-8df384f3da31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([197, 768])"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pooled_features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "cb0f0861-bbe1-4764-b015-9c0593b03b6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Calculate the mean of 'pooled_embeddings' along axis 1 to match the dimension of 'pooled_features'\n",
    "pooled_embeddings_reduced = torch.mean(pooled_embeddings, dim=-2, keepdim=True)  # Resulting shape: [1, 1, 768]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "829c99bb-3c73-487c-886c-0d32a1ccd8d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "pooled_features_reduced = torch.mean(pooled_features, dim=-2, keepdim=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "17aa06f4-f018-4670-a3e6-c71c4e2e1576",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([1, 768]), torch.Size([1, 768]))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pooled_embeddings_reduced.shape, pooled_features_reduced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "26c78797-8bf5-4e9a-afea-18ab61cabe10",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "3d90e80e-8f35-442c-9c18-01b8485da9f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = \"cuda\"\n",
    "pooled_features_reduced = pooled_features_reduced.to(device)\n",
    "pooled_embeddings_reduced = pooled_embeddings_reduced.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "765a7df0-3631-4d9d-993f-15ad5f966cf2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.0116], device='cuda:0')"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_sim = F.cosine_similarity(pooled_features_reduced, pooled_embeddings_reduced, dim=-1)\n",
    "cosine_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "90a4e07d-ff02-48b6-8127-e35b2532ce83",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define temperature parameter\n",
    "tau = 1  # You can adjust this value based on your problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "30f0f518-ce4c-422b-b8e8-00f9963f8301",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply softmax with temperature to both sets of embeddings\n",
    "softmax_pooled_features = F.softmax(pooled_features_reduced / tau, dim=1)\n",
    "softmax_pooled_embeddings = F.softmax(pooled_embeddings_reduced / tau, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "b160ae57-89fa-4743-873e-c3f5cc802af0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Entropy Loss: 6.652477741241455\n"
     ]
    }
   ],
   "source": [
    "# Calculate the cross-entropy loss\n",
    "loss = -torch.sum(softmax_pooled_embeddings * torch.log(softmax_pooled_features + 1e-10)) / softmax_pooled_embeddings.size(0)\n",
    "\n",
    "# Print or use the loss as needed\n",
    "print(\"Cross-Entropy Loss:\", loss.item())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9927317-b1d3-4851-b38f-021293498a84",
   "metadata": {},
   "source": [
    "# Full training code from here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "1dfa7a38-fcb5-4f42-a0b5-d7a7c0badccc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "d0fc6d33-7059-4635-8def-543201032c31",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import ViTFeatureExtractor, ViTModel\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "class VideoEncoder(nn.Module):\n",
    "    def __init__(self, pretrained_model_name, hidden_size):\n",
    "        super(VideoEncoder, self).__init__()\n",
    "        self.feature_extractor = ViTFeatureExtractor.from_pretrained(pretrained_model_name)\n",
    "        self.vit_model = ViTModel.from_pretrained(pretrained_model_name)\n",
    "        self.hidden_size = hidden_size\n",
    "    \n",
    "    def forward(self, video_frames):\n",
    "        # video_frames: (batch_size, num_frames, channels, height, width)\n",
    "        \n",
    "        batch_size, num_frames, _, _, _ = video_frames.size()\n",
    "        \n",
    "        # Reshape video_frames to (batch_size * num_frames, channels, height, width)\n",
    "        video_frames = video_frames.view(-1, *video_frames.shape[2:])\n",
    "        \n",
    "        # Extract features using ViT\n",
    "        inputs = self.feature_extractor(images=video_frames, return_tensors=\"pt\")\n",
    "        inputs = {key: value.to(video_frames.device) for key, value in inputs.items()}\n",
    "        \n",
    "        with torch.no_grad():\n",
    "            outputs = self.vit_model(**inputs)\n",
    "        \n",
    "        # Extract the features from the model's output\n",
    "        features = outputs.last_hidden_state  # (batch_size * num_frames, seq_len, hidden_size)\n",
    "        \n",
    "        # Reshape features to (batch_size, num_frames, seq_len, hidden_size)\n",
    "        features = features.view(batch_size, num_frames, *features.shape[1:])\n",
    "        \n",
    "        return features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "65fa363b-c347-4555-8301-52b6917c01b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextEncoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_size, hidden_size):\n",
    "        super(TextEncoder, self).__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_size)\n",
    "        self.rnn = nn.LSTM(embedding_size, hidden_size, batch_first=True)\n",
    "    \n",
    "    def forward(self, captions):\n",
    "        # captions: (batch_size, seq_len)\n",
    "        embedded = self.embedding(captions)\n",
    "        output, _ = self.rnn(embedded)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "f5297714-04ef-4fe6-845e-21a3ba7da172",
   "metadata": {},
   "outputs": [],
   "source": [
    "class VideoCaptioningModel(nn.Module):\n",
    "    def __init__(self, video_encoder, text_encoder):\n",
    "        super(VideoCaptioningModel, self).__init__()\n",
    "        self.video_encoder = video_encoder\n",
    "        self.text_encoder = text_encoder\n",
    "\n",
    "    def forward(self, video_features, captions):\n",
    "        video_encoded = self.video_encoder(video_features)\n",
    "        text_encoded = self.text_encoder(captions)\n",
    "        similarity = sim_func(video_encoded, text_encoded)\n",
    "        return similarity\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "f19612e6-d8bc-4489-9293-cc5ad91aa987",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import cv2\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms\n",
    "from PIL import Image\n",
    "\n",
    "# Define a custom dataset class for video-caption pairs\n",
    "class VideoCaptionDataset(Dataset):\n",
    "    def __init__(self, json_path, video_folder, transform=None):\n",
    "        self.video_folder = video_folder\n",
    "        self.transform = transform\n",
    "        self.data = self.load_json_data(json_path)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        video_info = self.data[idx]\n",
    "        video_id = video_info['video_id']\n",
    "        video_path = os.path.join(slef.video_folder, f'{video_id}.mp4')\n",
    "        captions = video_info['captions']\n",
    "\n",
    "        # Load video frames and apply transformations\n",
    "        video_frames = self.load_video_frames(video_path)\n",
    "\n",
    "        if self.transform:\n",
    "            video_frames = [self.transform(frame) for frame in video_frames]\n",
    "\n",
    "        return video_frames, captions\n",
    "\n",
    "    def load_json_data(self, json_path):\n",
    "        with open(json_path, 'r') as json_file:\n",
    "            data = json.load(json_file)\n",
    "        return data\n",
    "\n",
    "    def load_video_frames(self, video_path):\n",
    "        frames = []\n",
    "        cap = cv2.VideoCapture(videopath)\n",
    "\n",
    "        while cap.isOpened():\n",
    "            ret, frame = cap.read()\n",
    "            if not ret:\n",
    "                break\n",
    "            # Convert frame to PIL image\n",
    "            frame_pil = Image.fromarray(cv2.cvtColor(frame, cv2.BGR2RGB))\n",
    "            frames.append(frame_pil)\n",
    "\n",
    "        cap.release()\n",
    "        return frames\n",
    "\n",
    "# Define transformations for video frames (you can customize these)\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((224, 224)),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "])\n",
    "\n",
    "# Define paths and create data loaders for training and validation\n",
    "json_path = 'train_val_annotation/train_val_videodatainfo.json'  # Path to your JSON file\n",
    "video_folder = 'TrainValVideo'  # Path to the folder containing video files\n",
    "\n",
    "dataset = VideoCaptionDataset(json_path, video_folder, transform=transform)\n",
    "\n",
    "# Split the dataset into training and validation\n",
    "train_size = int(0.8 * len(dataset))\n",
    "val_size = len(dataset) - train_size\n",
    "\n",
    "train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, val_size])\n",
    "\n",
    "# Create data loaders\n",
    "batch_size = 32  # Adjust as needed\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "val_loader = DataLoader(val_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "eb4969af-b525-468a-ba90-f6952d416dd4",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'video_encoder' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[81], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m model \u001b[38;5;241m=\u001b[39m VideoCaptioningModel(\u001b[43mvideo_encoder\u001b[49m, text_encoder)\n\u001b[1;32m      2\u001b[0m criterion \u001b[38;5;241m=\u001b[39m nn\u001b[38;5;241m.\u001b[39mMSELoss()  \u001b[38;5;66;03m# You can use any suitable loss function\u001b[39;00m\n\u001b[1;32m      3\u001b[0m optimizer \u001b[38;5;241m=\u001b[39m optim\u001b[38;5;241m.\u001b[39mAdam(model\u001b[38;5;241m.\u001b[39mparameters(), lr\u001b[38;5;241m=\u001b[39mlearning_rate)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'video_encoder' is not defined"
     ]
    }
   ],
   "source": [
    "model = VideoCaptioningModel(video_encoder, text_encoder)\n",
    "criterion = nn.MSELoss()  # You can use any suitable loss function\n",
    "optimizer = optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ad7dc73",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 10\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "    model.train()\n",
    "    total_loss = 0.0\n",
    "\n",
    "    for batch in dataloader_train:\n",
    "        video_features, captions = batch\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        similarity = model(video_features, captions)\n",
    "        loss = criterion(similarity, your_target_similarity)\n",
    "        \n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        total_loss += loss.item()\n",
    "    \n",
    "    average_loss = total_loss / len(dataloader_train)\n",
    "    print(f'Epoch [{epoch+1}/{num_epochs}], Loss: {average_loss:.4f}')\n",
    "\n",
    "# Add validation loop and early stopping as needed\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88e219f2-1945-4861-921e-7bc52790b00f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}