log_to_gpt.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import time\n",
    "import openai\n",
    "\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def link_snippets(logs):\n",
    "    # Load messages array\n",
    "    messages = logs['messages']\n",
    "\n",
    "    # Create a string combining messages\n",
    "    refactored_msgs = []\n",
    "    for message in messages:\n",
    "        msg = ''\n",
    "        # Concatenate ['author']['name']: ['content']\n",
    "        msg += f\"{message['author']['name']}: {message['content']}\"\n",
    "\n",
    "        # Concatenate title and description for each embed\n",
    "        for embed in message['embeds']:\n",
    "            desc = embed['description']\n",
    "            # strip all urls from the description\n",
    "            for word in desc.split():\n",
    "                if 'http' in word:\n",
    "                    desc = desc.replace(word, '')\n",
    "\n",
    "            msg += f\"\\nLink: \\\"{embed['title']}\\\" Description: \\\"{desc}\\\"\"\n",
    "\n",
    "        refactored_msgs.append(msg)\n",
    "\n",
    "    link_indices = []\n",
    "    for i, message in enumerate(refactored_msgs):\n",
    "        if 'http' in message:\n",
    "            link_indices.append(i)\n",
    "\n",
    "    link_contexts = []\n",
    "    for index in link_indices:\n",
    "        link_context = ''\n",
    "        link_context_count = 0\n",
    "        # Check forward\n",
    "        for i in range(index, min(index+17, len(refactored_msgs))):\n",
    "            if 'http' in refactored_msgs[i]:\n",
    "                break\n",
    "            link_context += refactored_msgs[i] + '\\n\\n\\n'\n",
    "            link_context_count += 1\n",
    "            if link_context_count >= 16:\n",
    "                break\n",
    "        link_context += refactored_msgs[index] + '\\n\\n\\n'\n",
    "        # Check backward in reverse\n",
    "        for i in range(index-1, max(index-17, -1), -1):\n",
    "            if 'http' in refactored_msgs[i]:\n",
    "                break\n",
    "            link_context += refactored_msgs[i] + '\\n\\n\\n'\n",
    "            link_context_count += 1\n",
    "            if link_context_count >= 16:\n",
    "                break\n",
    "        link_contexts.append(link_context)\n",
    "\n",
    "    return link_contexts\n",
    "\n",
    "\n",
    "# call link_snippets for each json file in ./discord_logs, and concatenate the resulting array to ./links.json\n",
    "def link_snippets_all():\n",
    "    link_contexts = []\n",
    "    for filename in os.listdir('./discord_logs'):\n",
    "        if filename.endswith('.json'):\n",
    "            with open(f'./discord_logs/{filename}', 'r') as f:\n",
    "                logs = json.load(f)\n",
    "                link_contexts += link_snippets(logs)\n",
    "    with open('./links.json', 'w') as f:\n",
    "        json.dump(link_contexts, f)\n",
    "\n",
    "link_snippets_all()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load links.json, then, for each string in the list, generate a summary with gpt3, and add the link context + summary pair to a new json file called link_pairs.json\n",
    "\n",
    "def summarize(context):\n",
    "  return openai.ChatCompletion.create(\n",
    "      model=\"gpt-3.5-turbo\",\n",
    "      messages=[\n",
    "        {\n",
    "          \"role\": \"system\",\n",
    "          \"content\": \"I'd like you to help the user summarize the context of linked content in the chat snippet.\"\n",
    "        },\n",
    "        {\n",
    "          \"role\": \"user\",\n",
    "          \"content\": \"snippet:\\n```hazelwillow: https://thenextweb.com/news/new-experiment-demonstrates-reality-might-actually-be-real @Goldsteel, I don\\'t know if this falls in the scope of your studies, but I figured I\\'d ask if this is something you\\'d be willing to lecture us about in VRC at some point?\\nLink: \\\"New experiment demonstrates that reality might actually be real\\\" Description: \\\"What if reality wasn\\'t real? Can we prove it? A team of physicists may have just brought us a little closer to the answer.\\\"\\n\\n\\nhazelwillow: Exploring the intersection of atypical neurology and BCIs, and ethical considerations as to development of BCIs for minority populations? Panel discussion style\\n\\n\\nyellow0720: :duckthumbs:\\n\\n\\nyellow0720: hamp appreciation presentation owo\\n\\n\\n_hamp: Do you have a project, an idea, or just something that you think is cool and interesting, that you\\'d like to give a short talk or a presentation on at one of our meetups? If so then please share your idea briefly here, or send it to me via DM. \\n\\nYour presentation can be about anything that is at least somewhat related to transhumanism. Short presentations around 3-7 minutes long are usually preferred. Please don\\'t worry too much about your presentation not being good enough, even super simple talks are very welcome!\\n\\nIf we like your idea then we\\'ll gladly give you the floor at one of our meetups to give your presentation 😊```\\n\\n\\n\\n summarize the linked content in the chat snippet in 10 words or less. Do not mention the users involved in the conversation, only summarize information about the content. Avoid any direct copying existing link descriptions.\"\n",
    "        },\n",
    "        {\n",
    "          \"role\": \"assistant\",\n",
    "          \"content\": \"an experiment exploring the reality of existence\"\n",
    "        },\n",
    "        {\n",
    "          \"role\": \"user\",\n",
    "          \"content\": f\"snippet:\\n```{context}```\\n\\n\\n\\n summarize the linked content in the chat snippet in 10 words or less. Do not mention the users involved in the conversation, only summarize information about the content. Avoid any direct copying existing link descriptions.\"\n",
    "        }\n",
    "      ],\n",
    "      temperature=1,\n",
    "      max_tokens=256,\n",
    "      top_p=1,\n",
    "      frequency_penalty=0,\n",
    "      presence_penalty=0,\n",
    "      n=4\n",
    "    )\n",
    "\n",
    "with open('links.json') as f:\n",
    "  links = json.load(f)\n",
    "\n",
    "link_pairs = []\n",
    "for link in links[1:]:\n",
    "  response = None\n",
    "  while response is None:\n",
    "    try:\n",
    "      response = summarize(link)\n",
    "    except Exception as e:\n",
    "      print(\"error\")\n",
    "      print(e)\n",
    "      time.sleep(10)\n",
    "    \n",
    "  summaries = []\n",
    "  for choice in response.choices:\n",
    "    summaries.append(choice.message.content)\n",
    "  print(summaries)\n",
    "\n",
    "  link_pairs.append({\n",
    "      \"link\": link,\n",
    "      \"summaries\": summaries\n",
    "  })\n",
    "    \n",
    "  with open('link_pairs.json', 'w') as f:\n",
    "    json.dump(link_pairs, f)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43\n"
     ]
    }
   ],
   "source": [
    "print(len(link_pairs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}