devign.ipynb

{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": 27,
            "source": [
                "%load_ext autoreload\n",
                "%autoreload 2\n",
                "import refactorings\n",
                "from pathlib import Path\n",
                "import difflib\n",
                "import random\n",
                "import pandas as pd\n",
                "import random\n",
                "import tqdm.notebook as tqdm\n",
                "import tempfile\n",
                "from contextlib import redirect_stdout\n",
                "import pickle"
            ],
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": [
                        "The autoreload extension is already loaded. To reload it, use:\n",
                        "  %reload_ext autoreload\n"
                    ]
                }
            ],
            "metadata": {}
        },
        {
            "cell_type": "code",
            "execution_count": 49,
            "source": [
                "df = pd.read_json('/home/benja/work/weile-lab/thesis/ReVeal/data/ggnn_input/devign_cfg_full_text_files.json')\n",
                "i, functions = zip(*list(df.iterrows()))\n",
                "print(len(functions), 'samples')\n",
                "\n",
                "# functions = functions[:24]\n",
                "# print('cutting to', len(functions), 'samples')\n",
                "\n",
                "func_lines = functions[0][\"code\"].splitlines()\n",
                "print('Example:')\n",
                "print('\\n'.join(func_lines[:3]))\n",
                "print('...')\n",
                "print('\\n'.join(func_lines[-3:]))"
            ],
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": [
                        "26733 samples\n",
                        "Example:\n",
                        "static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,\n",
                        "                                int level, int strategy)\n",
                        " {\n",
                        "...\n",
                        "     buffer_reset(&vs->tight.zlib);\n",
                        "     return bytes;\n",
                        " }\n"
                    ]
                }
            ],
            "metadata": {}
        },
        {
            "cell_type": "code",
            "execution_count": 50,
            "source": [
                "def do_one(fn):\n",
                "    print(fn[\"file_name\"])\n",
                "    code = fn[\"code\"]\n",
                "    with open(tmp_file, 'w') as f:\n",
                "        f.write(code)\n",
                "    with factory.make_project(tmp_file) as project:\n",
                "        new_file, applied = project.apply_all(return_applied=True)\n",
                "        with open(new_file) as f:\n",
                "            new_fn = f.read()\n",
                "            # new_functions.append((new_fn, applied))\n",
                "            return (new_fn, applied)\n",
                "    # if len(new_functions) > 5000:\n",
                "    #     with open(f'new_functions.pkl.shard{shards}', 'wb') as f:\n",
                "    #         pickle.dump(new_functions, f)\n",
                "    #     shards += 1\n",
                "    #     new_functions = []\n",
                "\n",
                "import multiprocessing\n",
                "import itertools\n",
                "factory = refactorings.TransformationsFactory(refactorings.all_refactorings, refactorings.random_picker)\n",
                "with tempfile.TemporaryDirectory() as tmp_dir:\n",
                "    shards = 0\n",
                "    # new_functions = []\n",
                "    shard_len = 5000\n",
                "\n",
                "    tmp_dir = Path(tmp_dir)\n",
                "    tmp_file = tmp_dir / 'tmp.c'\n",
                "    print('Temporary file:', tmp_file)\n",
                "    \n",
                "    log_filename = 'log.txt'\n",
                "    print('Redirecting to', log_filename)\n",
                "    with open(log_filename, 'w') as f:\n",
                "        with redirect_stdout(f):\n",
                "            with multiprocessing.Pool(8) as p:\n",
                "                it = tqdm.tqdm(p.imap(do_one, functions), total=len(functions))\n",
                "                while True:\n",
                "                    new_functions = list(itertools.islice(it, shard_len))\n",
                "                    if len(new_functions) == 0:\n",
                "                        break\n",
                "                    else:\n",
                "                        shard_filename = f'new_functions.pkl.shard{shards}'\n",
                "                        print('Saving shard', shard_filename, 'size', len(new_functions))\n",
                "                        with open(shard_filename, 'wb') as f:\n",
                "                            pickle.dump(new_functions, f)\n",
                "                        shards += 1\n",
                "                        del new_functions"
            ],
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": [
                        "Temporary file: /tmp/tmp7_dviv0m/tmp.c\n",
                        "Redirecting to log.txt\n"
                    ]
                },
                {
                    "output_type": "display_data",
                    "data": {
                        "text/plain": [
                            "HBox(children=(IntProgress(value=0, max=26733), HTML(value='')))"
                        ],
                        "application/vnd.jupyter.widget-view+json": {
                            "version_major": 2,
                            "version_minor": 0,
                            "model_id": "79a2b57712e64d59bb7fc7f27400d126"
                        }
                    },
                    "metadata": {}
                },
                {
                    "output_type": "stream",
                    "name": "stderr",
                    "text": [
                        "Process ForkPoolWorker-31:\n",
                        "Process ForkPoolWorker-25:\n",
                        "Process ForkPoolWorker-30:\n",
                        "Process ForkPoolWorker-26:\n",
                        "Process ForkPoolWorker-29:\n",
                        "Traceback (most recent call last):\n",
                        "Traceback (most recent call last):\n",
                        "Process ForkPoolWorker-27:\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "Traceback (most recent call last):\n",
                        "Process ForkPoolWorker-28:\n",
                        "Process ForkPoolWorker-32:\n",
                        "Traceback (most recent call last):\n",
                        "Traceback (most recent call last):\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/pool.py\", line 125, in worker\n",
                        "    result = (True, func(*args, **kwds))\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/pool.py\", line 125, in worker\n",
                        "    result = (True, func(*args, **kwds))\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "Traceback (most recent call last):\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/pool.py\", line 125, in worker\n",
                        "    result = (True, func(*args, **kwds))\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/pool.py\", line 125, in worker\n",
                        "    result = (True, func(*args, **kwds))\n",
                        "  File \"<ipython-input-50-1fd5193f178a>\", line 8, in do_one\n",
                        "    new_file, applied = project.apply_all(return_applied=True)\n",
                        "  File \"<ipython-input-50-1fd5193f178a>\", line 8, in do_one\n",
                        "    new_file, applied = project.apply_all(return_applied=True)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/pool.py\", line 125, in worker\n",
                        "    result = (True, func(*args, **kwds))\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 92, in apply_all\n",
                        "    self.apply(t)\n",
                        "  File \"<ipython-input-50-1fd5193f178a>\", line 8, in do_one\n",
                        "    new_file, applied = project.apply_all(return_applied=True)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 92, in apply_all\n",
                        "    self.apply(t)\n",
                        "Traceback (most recent call last):\n",
                        "Traceback (most recent call last):\n",
                        "  File \"<ipython-input-50-1fd5193f178a>\", line 8, in do_one\n",
                        "    new_file, applied = project.apply_all(return_applied=True)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 92, in apply_all\n",
                        "    self.apply(t)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 57, in apply\n",
                        "    new_lines = t(self.c_filename, picker=self.picker, project=self.project, exclude=self.exclude, tmp_dir=self.tmp_dir).run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 57, in apply\n",
                        "    new_lines = t(self.c_filename, picker=self.picker, project=self.project, exclude=self.exclude, tmp_dir=self.tmp_dir).run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n",
                        "    self.run()\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/base.py\", line 39, in __init__\n",
                        "    self.joern = JoernInfo(self.c_file, self.info[\"project\"], self.info[\"exclude\"], self.info[\"tmp_dir\"])\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 57, in apply\n",
                        "    new_lines = t(self.c_filename, picker=self.picker, project=self.project, exclude=self.exclude, tmp_dir=self.tmp_dir).run()\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 92, in apply_all\n",
                        "    self.apply(t)\n",
                        "  File \"<ipython-input-50-1fd5193f178a>\", line 8, in do_one\n",
                        "    new_file, applied = project.apply_all(return_applied=True)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/base.py\", line 39, in __init__\n",
                        "    self.joern = JoernInfo(self.c_file, self.info[\"project\"], self.info[\"exclude\"], self.info[\"tmp_dir\"])\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/process.py\", line 108, in run\n",
                        "    self._target(*self._args, **self._kwargs)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/pool.py\", line 125, in worker\n",
                        "    result = (True, func(*args, **kwds))\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/joern.py\", line 7, in __init__\n",
                        "    self.g = cpg.parse(tmp_dir, project, c_file, exclude)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/base.py\", line 40, in __init__\n",
                        "    self.srcml_root = srcml.get_xml_from_file(self.c_file)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 57, in apply\n",
                        "    new_lines = t(self.c_filename, picker=self.picker, project=self.project, exclude=self.exclude, tmp_dir=self.tmp_dir).run()\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 92, in apply_all\n",
                        "    self.apply(t)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/joern.py\", line 7, in __init__\n",
                        "    self.g = cpg.parse(tmp_dir, project, c_file, exclude)\n",
                        "  File \"<ipython-input-50-1fd5193f178a>\", line 8, in do_one\n",
                        "    new_file, applied = project.apply_all(return_applied=True)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/cpg.py\", line 32, in parse\n",
                        "    proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/srcml.py\", line 85, in get_xml_from_file\n",
                        "    return xp(srcml(c_file), '//src:unit')[0]\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/cpg.py\", line 32, in parse\n",
                        "    proc = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/subprocess.py\", line 491, in run\n",
                        "    stdout, stderr = process.communicate(input, timeout=timeout)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/srcml.py\", line 65, in srcml\n",
                        "    proc = subprocess.run(args, env=srcml_env, capture_output=True)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/multiprocessing/pool.py\", line 125, in worker\n",
                        "    result = (True, func(*args, **kwds))\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/project.py\", line 92, in apply_all\n",
                        "    self.apply(t)\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/subprocess.py\", line 491, in run\n",
                        "    stdout, stderr = process.communicate(input, timeout=timeout)\n",
                        "  File \"/home/benja/work/weile-lab/thesis/cfactor/refactorings/base.py\", line 39, in __init__\n",
                        "    self.joern = JoernInfo(self.c_file, self.info[\"project\"], self.info[\"exclude\"], self.info[\"tmp_dir\"])\n",
                        "  File \"/home/benja/miniconda3/lib/python3.8/subprocess.py\", line 1011, in communicate\n",
                        "    stdout = self.stdout.read()\n"
                    ]
                },
                {
                    "output_type": "error",
                    "ename": "KeyboardInterrupt",
                    "evalue": "",
                    "traceback": [
                        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
                        "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
                        "\u001b[0;32m~/miniconda3/lib/python3.8/multiprocessing/pool.py\u001b[0m in \u001b[0;36mnext\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    850\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 851\u001b[0;31m                 \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_items\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopleft\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    852\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mIndexError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
                        "\u001b[0;31mIndexError\u001b[0m: pop from an empty deque",
                        "\nDuring handling of the above exception, another exception occurred:\n",
                        "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
                        "\u001b[0;32m<ipython-input-50-1fd5193f178a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     36\u001b[0m                 \u001b[0mit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdo_one\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfunctions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunctions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m                 \u001b[0;32mwhile\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 38\u001b[0;31m                     \u001b[0mnew_functions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitertools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mislice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mit\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshard_len\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     39\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_functions\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     40\u001b[0m                         \u001b[0;32mbreak\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
                        "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/tqdm/notebook.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    220\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 222\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtqdm_notebook\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__iter__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    223\u001b[0m                 \u001b[0;31m# return super(tqdm...) will not catch exception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    224\u001b[0m                 \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
                        "\u001b[0;32m~/miniconda3/lib/python3.8/site-packages/tqdm/std.py\u001b[0m in \u001b[0;36m__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1079\u001b[0m             \"\"\"), fp_write=getattr(self.fp, 'write', sys.stderr.write))\n\u001b[1;32m   1080\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1081\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miterable\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1082\u001b[0m             \u001b[0;32myield\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1083\u001b[0m             \u001b[0;31m# Update and possibly print the progressbar.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
                        "\u001b[0;32m~/miniconda3/lib/python3.8/multiprocessing/pool.py\u001b[0m in \u001b[0;36mnext\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    854\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pool\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    855\u001b[0m                     \u001b[0;32mraise\u001b[0m \u001b[0mStopIteration\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 856\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cond\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    857\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    858\u001b[0m                     \u001b[0mitem\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_items\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpopleft\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
                        "\u001b[0;32m~/miniconda3/lib/python3.8/threading.py\u001b[0m in \u001b[0;36mwait\u001b[0;34m(self, timeout)\u001b[0m\n\u001b[1;32m    300\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m    \u001b[0;31m# restore state no matter what (e.g., KeyboardInterrupt)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    301\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mtimeout\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 302\u001b[0;31m                 \u001b[0mwaiter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0macquire\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    303\u001b[0m                 \u001b[0mgotit\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    304\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
                        "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
                    ]
                }
            ],
            "metadata": {}
        },
        {
            "cell_type": "code",
            "execution_count": 34,
            "source": [
                "import difflib\n",
                "for i in range(len(new_functions)):\n",
                "    print('Applied:', [x.__name__ for x in new_functions[i][1]])\n",
                "    print(''.join(difflib.unified_diff(functions[i].splitlines(keepends=True), new_functions[i][0].splitlines(keepends=True))))"
            ],
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": [
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -1,8 +1,8 @@\n",
                        " static int tight_compress_data(VncState *vs, int stream_id, size_t bytes,\n",
                        "                                 int level, int strategy)\n",
                        "  {\n",
                        "-     z_streamp zstream = &vs->tight.stream[stream_id];\n",
                        "      int previous_out;\n",
                        "+     z_streamp fungus = &vs->tight.stream[stream_id];\n",
                        "      if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) {\n",
                        "          vnc_write(vs, vs->tight.tight.buffer, vs->tight.tight.offset);\n",
                        "          return bytes;\n",
                        "@@ -13,19 +13,20 @@\n",
                        "      /* reserve memory in output buffer */\n",
                        "      buffer_reserve(&vs->tight.zlib, bytes + 64);\n",
                        "      /* set pointers */\n",
                        "-     zstream->next_in = vs->tight.tight.buffer;\n",
                        "-     zstream->avail_in = vs->tight.tight.offset;\n",
                        "-     zstream->next_out = vs->tight.zlib.buffer + vs->tight.zlib.offset;\n",
                        "-     zstream->avail_out = vs->tight.zlib.capacity - vs->tight.zlib.offset;\n",
                        "-     zstream->data_type = Z_BINARY;\n",
                        "-     previous_out = zstream->total_out;\n",
                        "+     fungus->next_in = vs->tight.tight.buffer;\n",
                        "+     fungus->avail_in = vs->tight.tight.offset;\n",
                        "+     fungus->next_out = vs->tight.zlib.buffer + vs->tight.zlib.offset;\n",
                        "+     fungus->avail_out = vs->tight.zlib.capacity - vs->tight.zlib.offset;\n",
                        "+     fungus->data_type = Z_BINARY;\n",
                        "+     previous_out = fungus->total_out;\n",
                        "      /* start encoding */\n",
                        "-     if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) {\n",
                        "+     if (deflate(fungus, Z_SYNC_FLUSH) != Z_OK) {\n",
                        "          fprintf(stderr, \"VNC: error during tight compression\\n\");\n",
                        "          return -1;\n",
                        "      }\n",
                        "-     vs->tight.zlib.offset = vs->tight.zlib.capacity - zstream->avail_out;\n",
                        "-     bytes = zstream->total_out - previous_out;\n",
                        "+     char mungus = 'a';\n",
                        "+     vs->tight.zlib.offset = vs->tight.zlib.capacity - fungus->avail_out;\n",
                        "+     bytes = fungus->total_out - previous_out;\n",
                        "      tight_send_compact_size(vs, bytes);\n",
                        "      vnc_write(vs, vs->tight.zlib.buffer, bytes);\n",
                        "      buffer_reset(&vs->tight.zlib);\n",
                        "\n",
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -2,12 +2,12 @@\n",
                        "                 unsigned long start, unsigned long end,\n",
                        "                 int prot, int flags, int fd, unsigned long offset)\n",
                        "  {\n",
                        "-     unsigned long host_end, ret, addr;\n",
                        "      int prot1, prot_new;\n",
                        "-     host_end = host_start + qemu_host_page_size;\n",
                        "+     unsigned long fungus, ret, addr;\n",
                        "+     fungus = host_start + qemu_host_page_size;\n",
                        "      /* get the protection of the target pages outside the mapping */\n",
                        "      prot1 = 0;\n",
                        "-     for(addr = host_start; addr < host_end; addr++) {\n",
                        "+     for(addr = host_start; addr < fungus; addr++) {\n",
                        "          if (addr < start || addr >= end)\n",
                        "              prot1 |= page_get_flags(addr);\n",
                        "      }\n",
                        "@@ -40,6 +40,7 @@\n",
                        "              mprotect((void *)host_start, qemu_host_page_size, prot_new);\n",
                        "      } else {\n",
                        "          /* just update the protection */\n",
                        "+         char mungus = 'a';\n",
                        "          if (prot_new != prot1) {\n",
                        "              mprotect((void *)host_start, qemu_host_page_size, prot_new);\n",
                        "          }\n",
                        "\n",
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -1,13 +1,14 @@\n",
                        " void restore_boot_order(void *opaque)\n",
                        "  {\n",
                        "-     char *normal_boot_order = opaque;\n",
                        "      static int first = 1;\n",
                        "+     char *fungus = opaque;\n",
                        "      /* Restore boot order and remove ourselves after the first boot */\n",
                        "      if (first) {\n",
                        "          first = 0;\n",
                        "          return;\n",
                        "      }\n",
                        "-     qemu_boot_set(normal_boot_order);\n",
                        "-     qemu_unregister_reset(restore_boot_order, normal_boot_order);\n",
                        "-     g_free(normal_boot_order);\n",
                        "+     qemu_boot_set(fungus);\n",
                        "+     char mungus = 'a';\n",
                        "+     qemu_unregister_reset(restore_boot_order, fungus);\n",
                        "+     g_free(fungus);\n",
                        "  }\n",
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -1,10 +1,11 @@\n",
                        " uint64_t helper_cmpbge (uint64_t op1, uint64_t op2)\n",
                        "  {\n",
                        "+     int i;\n",
                        "      uint8_t opa, opb, res;\n",
                        "-     int i;\n",
                        "      res = 0;\n",
                        "      for (i = 0; i < 7; i++) {\n",
                        "          opa = op1 >> (i * 8);\n",
                        "+         char fungus = 'a';\n",
                        "          opb = op2 >> (i * 8);\n",
                        "          if (opa >= opb)\n",
                        "              res |= 1 << i;\n",
                        "\n",
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -1,23 +1,23 @@\n",
                        " static int64_t coroutine_fn qcow2_co_get_block_status(BlockDriverState *bs,\n",
                        "          int64_t sector_num, int nb_sectors, int *pnum, BlockDriverState **file)\n",
                        "  {\n",
                        "-     BDRVQcow2State *s = bs->opaque;\n",
                        "+     BDRVQcow2State *fungus = bs->opaque;\n",
                        "      uint64_t cluster_offset;\n",
                        "      int index_in_cluster, ret;\n",
                        "      unsigned int bytes;\n",
                        "+     qemu_co_mutex_lock(&fungus->lock);\n",
                        "+     bytes = MIN(INT_MAX, nb_sectors * BDRV_SECTOR_SIZE);\n",
                        "      int64_t status = 0;\n",
                        "-     bytes = MIN(INT_MAX, nb_sectors * BDRV_SECTOR_SIZE);\n",
                        "-     qemu_co_mutex_lock(&s->lock);\n",
                        "      ret = qcow2_get_cluster_offset(bs, sector_num << 9, &bytes,\n",
                        "                                     &cluster_offset);\n",
                        "-     qemu_co_mutex_unlock(&s->lock);\n",
                        "+     qemu_co_mutex_unlock(&fungus->lock);\n",
                        "      if (ret < 0) {\n",
                        "          return ret;\n",
                        "      }\n",
                        "      *pnum = bytes >> BDRV_SECTOR_BITS;\n",
                        "      if (cluster_offset != 0 && ret != QCOW2_CLUSTER_COMPRESSED &&\n",
                        "-         !s->cipher) {\n",
                        "-         index_in_cluster = sector_num & (s->cluster_sectors - 1);\n",
                        "+         !fungus->cipher) {\n",
                        "+         index_in_cluster = sector_num & (fungus->cluster_sectors - 1);\n",
                        "          cluster_offset |= (index_in_cluster << BDRV_SECTOR_BITS);\n",
                        "          *file = bs->file->bs;\n",
                        "          status |= BDRV_BLOCK_OFFSET_VALID | cluster_offset;\n",
                        "@@ -25,6 +25,7 @@\n",
                        "      if (ret == QCOW2_CLUSTER_ZERO_PLAIN || ret == QCOW2_CLUSTER_ZERO_ALLOC) {\n",
                        "          status |= BDRV_BLOCK_ZERO;\n",
                        "      } else if (ret != QCOW2_CLUSTER_UNALLOCATED) {\n",
                        "+         char mungus = 'a';\n",
                        "          status |= BDRV_BLOCK_DATA;\n",
                        "      }\n",
                        "      return status;\n",
                        "\n",
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -4,6 +4,7 @@\n",
                        "      if((uint64_t)atom.size > (1<<30))\n",
                        "          return -1;\n",
                        "      av_free(st->codec->extradata);\n",
                        "+     char fungus = 'a';\n",
                        "      st->codec->extradata = av_mallocz(atom.size + FF_INPUT_BUFFER_PADDING_SIZE);\n",
                        "      if (!st->codec->extradata)\n",
                        "          return AVERROR(ENOMEM);\n",
                        "\n",
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -2,4 +2,5 @@\n",
                        "  {\n",
                        "      /* 1k pages not implemented */\n",
                        "      env->CP0_PageMask = arg1 & (0x1FFFFFFF & (TARGET_PAGE_MASK << 1));\n",
                        "+     char fungus = 'a';\n",
                        "  }\n",
                        "Applied: ['InsertNoop', 'LoopExchange', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -4,8 +4,8 @@\n",
                        "      int i, b = 0;\n",
                        "      uint32_t cm[2] = { (1 << f->blocks) - 1, (1 << f->blocks) - 1 };\n",
                        "      const int band_size = ff_celt_freq_range[band] << f->size;\n",
                        "+     float dist, cost, err_x = 0.0f, err_y = 0.0f;\n",
                        "      float buf[176 * 2], lowband_scratch[176], norm1[176], norm2[176];\n",
                        "-     float dist, cost, err_x = 0.0f, err_y = 0.0f;\n",
                        "      float *X = buf;\n",
                        "      float *X_orig = f->block[0].coeffs + (ff_celt_freq_bands[band] << f->size);\n",
                        "      float *Y = (f->channels == 2) ? &buf[176] : NULL;\n",
                        "@@ -16,8 +16,8 @@\n",
                        "          memcpy(Y, Y_orig, band_size*sizeof(float));\n",
                        "      f->remaining2 = ((f->framebits << 3) - f->anticollapse_needed) - opus_rc_tell_frac(rc) - 1;\n",
                        "      if (band <= f->coded_bands - 1) {\n",
                        "-         int curr_balance = f->remaining / FFMIN(3, f->coded_bands - band);\n",
                        "-         b = av_clip_uintp2(FFMIN(f->remaining2 + 1, f->pulses[band] + curr_balance), 14);\n",
                        "+         int fungus = f->remaining / FFMIN(3, f->coded_bands - band);\n",
                        "+         b = av_clip_uintp2(FFMIN(f->remaining2 + 1, f->pulses[band] + fungus), 14);\n",
                        "      }\n",
                        "      if (f->dual_stereo) {\n",
                        "          pvq->encode_band(pvq, f, rc, band, X, NULL, band_size, b / 2, f->blocks, NULL,\n",
                        "@@ -28,9 +28,12 @@\n",
                        "          pvq->encode_band(pvq, f, rc, band, X, Y, band_size, b, f->blocks, NULL, f->size,\n",
                        "                           norm1, 0, 1.0f, lowband_scratch, cm[0] | cm[1]);\n",
                        "      }\n",
                        "-     for (i = 0; i < band_size; i++) {\n",
                        "+     i = 0 ;\n",
                        "+     while (i < band_size) {\n",
                        "          err_x += (X[i] - X_orig[i])*(X[i] - X_orig[i]);\n",
                        "+         char mungus = 'a';\n",
                        "          err_y += (Y[i] - Y_orig[i])*(Y[i] - Y_orig[i]);\n",
                        "+         i ++;\n",
                        "      }\n",
                        "      dist = sqrtf(err_x) + sqrtf(err_y);\n",
                        "      cost = OPUS_RC_CHECKPOINT_BITS(rc)/8.0f;\n",
                        "\n",
                        "Applied: ['InsertNoop', 'LoopExchange', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -2,10 +2,13 @@\n",
                        "  {\n",
                        "      int i, re, im, re1, im1;\n",
                        "      int16_t rot[MDCT_SAMPLES];\n",
                        "+     i = 0 ;\n",
                        "+     /* shift to simplify computations */\n",
                        "      IComplex x[MDCT_SAMPLES/4];\n",
                        "-     /* shift to simplify computations */\n",
                        "-     for (i = 0; i < MDCT_SAMPLES/4; i++)\n",
                        "+     while (i < MDCT_SAMPLES / 4) {\n",
                        "          rot[i] = -in[i + 3*MDCT_SAMPLES/4];\n",
                        "+         i ++;\n",
                        "+     }\n",
                        "      for (;i < MDCT_SAMPLES; i++)\n",
                        "          rot[i] =  in[i -   MDCT_SAMPLES/4];\n",
                        "      /* pre rotation */\n",
                        "@@ -14,6 +17,7 @@\n",
                        "          im = -((int)rot[MDCT_SAMPLES/2+2*i] - (int)rot[MDCT_SAMPLES/2-1-2*i]) >> 1;\n",
                        "          CMUL(x[i].re, x[i].im, re, im, -xcos1[i], xsin1[i]);\n",
                        "      }\n",
                        "+     char fungus = 'a';\n",
                        "      fft(x, MDCT_NBITS - 2);\n",
                        "      /* post rotation */\n",
                        "      for (i = 0; i < MDCT_SAMPLES/4; i++) {\n",
                        "\n",
                        "Applied: ['InsertNoop', 'PermuteStmt', 'RenameVariable']\n",
                        "--- \n",
                        "+++ \n",
                        "@@ -25,6 +25,7 @@\n",
                        "      vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0);\n",
                        "      vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0);\n",
                        "      vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0);\n",
                        "+     char fungus = 'a';\n",
                        "      vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL);\n",
                        "      vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0);\n",
                        "      vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0);\n",
                        "\n"
                    ]
                }
            ],
            "metadata": {}
        },
        {
            "cell_type": "code",
            "execution_count": 47,
            "source": [
                "import itertools\n",
                "it = iter([1, 2, 3, 4, 5])\n",
                "print(list(itertools.islice(it, 3)))\n",
                "print(list(itertools.islice(it, 3)))\n",
                "print(list(itertools.islice(it, 3)))"
            ],
            "outputs": [
                {
                    "output_type": "stream",
                    "name": "stdout",
                    "text": [
                        "[1, 2, 3]\n",
                        "[4, 5]\n",
                        "[]\n"
                    ]
                }
            ],
            "metadata": {}
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "source": [],
            "outputs": [],
            "metadata": {}
        }
    ],
    "metadata": {
        "orig_nbformat": 4,
        "language_info": {
            "name": "python",
            "version": "3.8.5",
            "mimetype": "text/x-python",
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "pygments_lexer": "ipython3",
            "nbconvert_exporter": "python",
            "file_extension": ".py"
        },
        "kernelspec": {
            "name": "python3",
            "display_name": "Python 3.8.5 64-bit ('base': conda)"
        },
        "interpreter": {
            "hash": "a535d873fe0e83797f6e6a2091476b53ca700b1c39e49c8cf317620b8a39cd7e"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}