data-processing-utils.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Queries"
      ],
      "metadata": {
        "id": "Lhuo3jp7SDy5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data_user_welfare_surplus = '''\n",
        "SELECT \n",
        "    block_number,\n",
        "    `hash`,\n",
        "    transaction_index as `index`,\n",
        "    gas as gas_limit,\n",
        "    gas_price,\n",
        "    receipt_gas_used as gas_used,\n",
        "    max_fee_per_gas,\n",
        "    max_priority_fee_per_gas,\n",
        "    MAX(gas_price-max_priority_fee_per_gas) OVER (PARTITION BY block_number) AS base_fee,\n",
        "    receipt_status as status,\n",
        "    transaction_type as type\n",
        "FROM `bigquery-public-data.crypto_ethereum.transactions`\n",
        "WHERE DATE(block_timestamp) = \"2023-01-01\"\n",
        "\n",
        "-- priority fee is to compute using basefee\n",
        "'''\n",
        "\n",
        "data_sim_4844 = '''\n",
        "SELECT\n",
        "    block_number,\n",
        "    `hash`,\n",
        "    transaction_index as `index`,\n",
        "    gas as gas_limit,\n",
        "    gas_price,\n",
        "    receipt_gas_used as gas_used,\n",
        "    max_fee_per_gas,\n",
        "    max_priority_fee_per_gas,\n",
        "    MAX(gas_price-max_priority_fee_per_gas) OVER (PARTITION BY block_number) AS base_fee,\n",
        "    receipt_status as status,\n",
        "    transaction_type as type,\n",
        "    CASE\n",
        "        WHEN (\n",
        "        from_address = \"0x6887246668a3b87f54deb3b94ba47a6f63f32985\" -- op sequencer address\n",
        "        AND to_address = \"0x5e4e65926ba27467555eb562121fac00d24e9dd2\" -- op CanonicalTransactionChain address\n",
        "        ) THEN \"op_data\"\n",
        "        WHEN (\n",
        "        from_address = \"0xc1b634853cb333d3ad8663715b08f41a3aec47cc\" -- arb batch submitter address\n",
        "        AND to_address = \"0x1c479675ad559dc151f6ec7ed3fbf8cee79582b6\" -- arb sequencer inbox\n",
        "        ) THEN \"arb_data\"\n",
        "        ELSE \"other\"\n",
        "    END AS data_tag,\n",
        "    (byte_length(input) / 2 - 1) AS data_bytes\n",
        "FROM `bigquery-public-data.crypto_ethereum.transactions`\n",
        "WHERE DATE(block_timestamp) = \"2023-02-24\"\n",
        "'''"
      ],
      "metadata": {
        "id": "FWzuMkqsSTZJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# 4844 Data for Simulation"
      ],
      "metadata": {
        "id": "g4geO3FzSIie"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "KxSqPfAeXW_K",
        "outputId": "282fd86c-33b0-4169-c47c-2853303759f8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   block_number                                               hash  index  \\\n",
              "0      16694584  0x971431be8deee130ff52febda56bdde09e3435e85d0f...     14   \n",
              "1      16694584  0x43c02349d35bfadb6709cf1cfb2c051a8d122056e2fa...    120   \n",
              "2      16694584  0xb5560d0f250edd82a4abd54b8258bef625b014b8d18f...     57   \n",
              "3      16694584  0xcd4ba9f4413390abd616255c2774b731f8d850973880...     84   \n",
              "4      16694584  0x07e6da1121e833e6f9ca9a43806786711c3301b0a0d8...     67   \n",
              "\n",
              "   gas_limit    gas_price  gas_used  max_fee_per_gas  \\\n",
              "0     159222  34885439765    104935             <NA>   \n",
              "1      48612  32801451293     29340      45954155418   \n",
              "2     124588  33301451293    119600      41403693704   \n",
              "3     233872  32801451293    157602      45954155418   \n",
              "4     250000  32919132298     34237             <NA>   \n",
              "\n",
              "   max_priority_fee_per_gas     base_fee  status  type data_tag  data_bytes  \n",
              "0                      <NA>  32301451293       1     0    other       132.0  \n",
              "1                 500000000  32301451293       1     2    other        68.0  \n",
              "2                1000000000  32301451293       1     2    other        36.0  \n",
              "3                 500000000  32301451293       1     2    other      1060.0  \n",
              "4                      <NA>  32301451293       1     0    other        68.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-cbc10606-c1f2-4ad6-bfce-104dd5db82a3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>block_number</th>\n",
              "      <th>hash</th>\n",
              "      <th>index</th>\n",
              "      <th>gas_limit</th>\n",
              "      <th>gas_price</th>\n",
              "      <th>gas_used</th>\n",
              "      <th>max_fee_per_gas</th>\n",
              "      <th>max_priority_fee_per_gas</th>\n",
              "      <th>base_fee</th>\n",
              "      <th>status</th>\n",
              "      <th>type</th>\n",
              "      <th>data_tag</th>\n",
              "      <th>data_bytes</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x971431be8deee130ff52febda56bdde09e3435e85d0f...</td>\n",
              "      <td>14</td>\n",
              "      <td>159222</td>\n",
              "      <td>34885439765</td>\n",
              "      <td>104935</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>other</td>\n",
              "      <td>132.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x43c02349d35bfadb6709cf1cfb2c051a8d122056e2fa...</td>\n",
              "      <td>120</td>\n",
              "      <td>48612</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>29340</td>\n",
              "      <td>45954155418</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>68.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0xb5560d0f250edd82a4abd54b8258bef625b014b8d18f...</td>\n",
              "      <td>57</td>\n",
              "      <td>124588</td>\n",
              "      <td>33301451293</td>\n",
              "      <td>119600</td>\n",
              "      <td>41403693704</td>\n",
              "      <td>1000000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0xcd4ba9f4413390abd616255c2774b731f8d850973880...</td>\n",
              "      <td>84</td>\n",
              "      <td>233872</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>157602</td>\n",
              "      <td>45954155418</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>1060.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x07e6da1121e833e6f9ca9a43806786711c3301b0a0d8...</td>\n",
              "      <td>67</td>\n",
              "      <td>250000</td>\n",
              "      <td>32919132298</td>\n",
              "      <td>34237</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>other</td>\n",
              "      <td>68.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cbc10606-c1f2-4ad6-bfce-104dd5db82a3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-cbc10606-c1f2-4ad6-bfce-104dd5db82a3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-cbc10606-c1f2-4ad6-bfce-104dd5db82a3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "from google.colab import auth\n",
        "from google.cloud import bigquery\n",
        "\n",
        "project = 'incrementai-proto'\n",
        "location = 'US'\n",
        "client = bigquery.Client(project=project, location=location)\n",
        "auth.authenticate_user()\n",
        "\n",
        "job = client.get_job('bquxjob_6e2a1255_186de4908d7')\n",
        "\n",
        "df = job.to_dataframe()\n",
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import statistics as stats"
      ],
      "metadata": {
        "id": "8kGQi8MoJmqD"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Approx max_fee and priority_fee for type 1 txns"
      ],
      "metadata": {
        "id": "Ih_QgPNSU5BF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "bin_labels = range(1,9)\n",
        "df['gas_bucket'] = pd.qcut(df['gas_used'], q=10, labels=bin_labels, duplicates='drop')"
      ],
      "metadata": {
        "id": "kI-AQuHbHy-T"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dfm = df[df.type==2][['block_number', 'gas_bucket', 'max_fee_per_gas', 'max_priority_fee_per_gas']].groupby(\n",
        "    ['block_number', 'gas_bucket']).apply(lambda x: pd.Series(dict(\n",
        "        max_fee_imputed = stats.median(x['max_fee_per_gas']),\n",
        "        max_priority_fee_imputed = stats.median(x['max_priority_fee_per_gas']),\n",
        "    ))).reset_index()\n",
        "dfm.tail(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "fFWBaCd9H_-L",
        "outputId": "f8f07127-58f0-4161-9f03-b1d7ffbc2318"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       block_number gas_bucket  max_fee_imputed  max_priority_fee_imputed\n",
              "56638      16701597          7     3.331738e+10              5.000000e+08\n",
              "56639      16701597          8     3.331738e+10              5.000000e+08\n",
              "56640      16701598          1     3.530946e+10              5.000000e+08\n",
              "56641      16701598          2     4.011138e+10              1.250000e+09\n",
              "56642      16701598          3     3.893051e+10              7.500000e+08\n",
              "56643      16701598          4     3.553544e+10              5.000000e+08\n",
              "56644      16701598          5     3.553544e+10              5.000000e+08\n",
              "56645      16701598          6     3.553544e+10              5.000000e+08\n",
              "56646      16701598          7     3.331738e+10              5.000000e+08\n",
              "56647      16701598          8     3.553544e+10              5.000000e+08"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6eb9c4d2-40d4-47d1-b33a-11a01fef29e0\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>block_number</th>\n",
              "      <th>gas_bucket</th>\n",
              "      <th>max_fee_imputed</th>\n",
              "      <th>max_priority_fee_imputed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>56638</th>\n",
              "      <td>16701597</td>\n",
              "      <td>7</td>\n",
              "      <td>3.331738e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56639</th>\n",
              "      <td>16701597</td>\n",
              "      <td>8</td>\n",
              "      <td>3.331738e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56640</th>\n",
              "      <td>16701598</td>\n",
              "      <td>1</td>\n",
              "      <td>3.530946e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56641</th>\n",
              "      <td>16701598</td>\n",
              "      <td>2</td>\n",
              "      <td>4.011138e+10</td>\n",
              "      <td>1.250000e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56642</th>\n",
              "      <td>16701598</td>\n",
              "      <td>3</td>\n",
              "      <td>3.893051e+10</td>\n",
              "      <td>7.500000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56643</th>\n",
              "      <td>16701598</td>\n",
              "      <td>4</td>\n",
              "      <td>3.553544e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56644</th>\n",
              "      <td>16701598</td>\n",
              "      <td>5</td>\n",
              "      <td>3.553544e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56645</th>\n",
              "      <td>16701598</td>\n",
              "      <td>6</td>\n",
              "      <td>3.553544e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56646</th>\n",
              "      <td>16701598</td>\n",
              "      <td>7</td>\n",
              "      <td>3.331738e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>56647</th>\n",
              "      <td>16701598</td>\n",
              "      <td>8</td>\n",
              "      <td>3.553544e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6eb9c4d2-40d4-47d1-b33a-11a01fef29e0')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6eb9c4d2-40d4-47d1-b33a-11a01fef29e0 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6eb9c4d2-40d4-47d1-b33a-11a01fef29e0');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = df.merge(dfm, on=['block_number', 'gas_bucket'], how='inner')\n",
        "df.head(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "id": "67HFDLokKQ78",
        "outputId": "31dac657-7f47-45d1-ab83-49130ca0c23d"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   block_number                                               hash  index  \\\n",
              "0      16694584  0x971431be8deee130ff52febda56bdde09e3435e85d0f...     14   \n",
              "1      16694584  0xb5560d0f250edd82a4abd54b8258bef625b014b8d18f...     57   \n",
              "2      16694584  0xe292b6f7cef8c6abff30eba47996d4f6480191a96726...     36   \n",
              "3      16694584  0x41964d9d770c6a8256e88ae66cd010a386c0251f889d...    106   \n",
              "4      16694584  0x8fa3a033cd5b42cc2bf5ebba748fd60233f70ce2e60d...     47   \n",
              "5      16694584  0x998c815f1edd2b470629b6165cede9c55f756fc4e8ff...    113   \n",
              "6      16694584  0x5e9f1ab7d9f31b137b548f8d11b5366aed9c2e2cc48f...    132   \n",
              "7      16694584  0x81b678cf78e276a3b6a144790ef270afaed5decfb899...    104   \n",
              "8      16694584  0x3f7f7e3c5dea859d71d74d4cd5c4d5771d3f6e7200e7...     24   \n",
              "9      16694584  0x3b76d38e77f2c5a29e23fb0dc770094b7070d6e68243...     77   \n",
              "\n",
              "   gas_limit    gas_price  gas_used  max_fee_per_gas  \\\n",
              "0     159222  34885439765    104935             <NA>   \n",
              "1     124588  33301451293    119600      41403693704   \n",
              "2     116900  34301451293    110595      46000000000   \n",
              "3     116900  32801451293    110595      45954155418   \n",
              "4     200000  33410682710    105818      68482871585   \n",
              "5     116900  32801451293    110595      42182943234   \n",
              "6     116900  32418209449    110595      40540942256   \n",
              "7     116900  32801451293    110595      45954155418   \n",
              "8     116900  34301451293    110595      46000000000   \n",
              "9     116900  32801451293    110595      41573640362   \n",
              "\n",
              "   max_priority_fee_per_gas     base_fee  status  type data_tag  data_bytes  \\\n",
              "0                      <NA>  32301451293       1     0    other       132.0   \n",
              "1                1000000000  32301451293       1     2    other        36.0   \n",
              "2                2000000000  32301451293       1     2    other        36.0   \n",
              "3                 500000000  32301451293       1     2    other        36.0   \n",
              "4                1109231417  32301451293       1     2    other       164.0   \n",
              "5                 500000000  32301451293       1     2    other        36.0   \n",
              "6                 116758156  32301451293       1     2    other        36.0   \n",
              "7                 500000000  32301451293       1     2    other        36.0   \n",
              "8                2000000000  32301451293       1     2    other        36.0   \n",
              "9                 500000000  32301451293       1     2    other        36.0   \n",
              "\n",
              "  gas_bucket  max_fee_imputed  max_priority_fee_imputed  \n",
              "0          6     4.595416e+10               500000000.0  \n",
              "1          6     4.595416e+10               500000000.0  \n",
              "2          6     4.595416e+10               500000000.0  \n",
              "3          6     4.595416e+10               500000000.0  \n",
              "4          6     4.595416e+10               500000000.0  \n",
              "5          6     4.595416e+10               500000000.0  \n",
              "6          6     4.595416e+10               500000000.0  \n",
              "7          6     4.595416e+10               500000000.0  \n",
              "8          6     4.595416e+10               500000000.0  \n",
              "9          6     4.595416e+10               500000000.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-965776d6-ab1d-4eaa-b247-54f7c54f07c8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>block_number</th>\n",
              "      <th>hash</th>\n",
              "      <th>index</th>\n",
              "      <th>gas_limit</th>\n",
              "      <th>gas_price</th>\n",
              "      <th>gas_used</th>\n",
              "      <th>max_fee_per_gas</th>\n",
              "      <th>max_priority_fee_per_gas</th>\n",
              "      <th>base_fee</th>\n",
              "      <th>status</th>\n",
              "      <th>type</th>\n",
              "      <th>data_tag</th>\n",
              "      <th>data_bytes</th>\n",
              "      <th>gas_bucket</th>\n",
              "      <th>max_fee_imputed</th>\n",
              "      <th>max_priority_fee_imputed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x971431be8deee130ff52febda56bdde09e3435e85d0f...</td>\n",
              "      <td>14</td>\n",
              "      <td>159222</td>\n",
              "      <td>34885439765</td>\n",
              "      <td>104935</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>other</td>\n",
              "      <td>132.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0xb5560d0f250edd82a4abd54b8258bef625b014b8d18f...</td>\n",
              "      <td>57</td>\n",
              "      <td>124588</td>\n",
              "      <td>33301451293</td>\n",
              "      <td>119600</td>\n",
              "      <td>41403693704</td>\n",
              "      <td>1000000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0xe292b6f7cef8c6abff30eba47996d4f6480191a96726...</td>\n",
              "      <td>36</td>\n",
              "      <td>116900</td>\n",
              "      <td>34301451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>46000000000</td>\n",
              "      <td>2000000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x41964d9d770c6a8256e88ae66cd010a386c0251f889d...</td>\n",
              "      <td>106</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>45954155418</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x8fa3a033cd5b42cc2bf5ebba748fd60233f70ce2e60d...</td>\n",
              "      <td>47</td>\n",
              "      <td>200000</td>\n",
              "      <td>33410682710</td>\n",
              "      <td>105818</td>\n",
              "      <td>68482871585</td>\n",
              "      <td>1109231417</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>164.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x998c815f1edd2b470629b6165cede9c55f756fc4e8ff...</td>\n",
              "      <td>113</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>42182943234</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x5e9f1ab7d9f31b137b548f8d11b5366aed9c2e2cc48f...</td>\n",
              "      <td>132</td>\n",
              "      <td>116900</td>\n",
              "      <td>32418209449</td>\n",
              "      <td>110595</td>\n",
              "      <td>40540942256</td>\n",
              "      <td>116758156</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x81b678cf78e276a3b6a144790ef270afaed5decfb899...</td>\n",
              "      <td>104</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>45954155418</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x3f7f7e3c5dea859d71d74d4cd5c4d5771d3f6e7200e7...</td>\n",
              "      <td>24</td>\n",
              "      <td>116900</td>\n",
              "      <td>34301451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>46000000000</td>\n",
              "      <td>2000000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x3b76d38e77f2c5a29e23fb0dc770094b7070d6e68243...</td>\n",
              "      <td>77</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>41573640362</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-965776d6-ab1d-4eaa-b247-54f7c54f07c8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-965776d6-ab1d-4eaa-b247-54f7c54f07c8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-965776d6-ab1d-4eaa-b247-54f7c54f07c8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df['max_fee'] = df.apply(lambda x: x['max_fee_per_gas'] if x['type']==2 else max(x['max_fee_imputed'], x['gas_price']), axis=1) # to avoid negative surplus take the max of imputed max fee and price\n",
        "df['max_priority_fee'] = df.apply(lambda x: x['max_priority_fee_per_gas'] if x['type']==2 else x['max_priority_fee_imputed'], axis=1)\n",
        "df.head(10)"
      ],
      "metadata": {
        "id": "SMDRThPfK5ka",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 0
        },
        "outputId": "6b36858a-080c-4ee6-8ff3-57aac7c9f8de"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   block_number                                               hash  index  \\\n",
              "0      16694584  0x971431be8deee130ff52febda56bdde09e3435e85d0f...     14   \n",
              "1      16694584  0xb5560d0f250edd82a4abd54b8258bef625b014b8d18f...     57   \n",
              "2      16694584  0xe292b6f7cef8c6abff30eba47996d4f6480191a96726...     36   \n",
              "3      16694584  0x41964d9d770c6a8256e88ae66cd010a386c0251f889d...    106   \n",
              "4      16694584  0x8fa3a033cd5b42cc2bf5ebba748fd60233f70ce2e60d...     47   \n",
              "5      16694584  0x998c815f1edd2b470629b6165cede9c55f756fc4e8ff...    113   \n",
              "6      16694584  0x5e9f1ab7d9f31b137b548f8d11b5366aed9c2e2cc48f...    132   \n",
              "7      16694584  0x81b678cf78e276a3b6a144790ef270afaed5decfb899...    104   \n",
              "8      16694584  0x3f7f7e3c5dea859d71d74d4cd5c4d5771d3f6e7200e7...     24   \n",
              "9      16694584  0x3b76d38e77f2c5a29e23fb0dc770094b7070d6e68243...     77   \n",
              "\n",
              "   gas_limit    gas_price  gas_used  max_fee_per_gas  \\\n",
              "0     159222  34885439765    104935             <NA>   \n",
              "1     124588  33301451293    119600      41403693704   \n",
              "2     116900  34301451293    110595      46000000000   \n",
              "3     116900  32801451293    110595      45954155418   \n",
              "4     200000  33410682710    105818      68482871585   \n",
              "5     116900  32801451293    110595      42182943234   \n",
              "6     116900  32418209449    110595      40540942256   \n",
              "7     116900  32801451293    110595      45954155418   \n",
              "8     116900  34301451293    110595      46000000000   \n",
              "9     116900  32801451293    110595      41573640362   \n",
              "\n",
              "   max_priority_fee_per_gas     base_fee  status  type data_tag  data_bytes  \\\n",
              "0                      <NA>  32301451293       1     0    other       132.0   \n",
              "1                1000000000  32301451293       1     2    other        36.0   \n",
              "2                2000000000  32301451293       1     2    other        36.0   \n",
              "3                 500000000  32301451293       1     2    other        36.0   \n",
              "4                1109231417  32301451293       1     2    other       164.0   \n",
              "5                 500000000  32301451293       1     2    other        36.0   \n",
              "6                 116758156  32301451293       1     2    other        36.0   \n",
              "7                 500000000  32301451293       1     2    other        36.0   \n",
              "8                2000000000  32301451293       1     2    other        36.0   \n",
              "9                 500000000  32301451293       1     2    other        36.0   \n",
              "\n",
              "  gas_bucket  max_fee_imputed  max_priority_fee_imputed       max_fee  \\\n",
              "0          6     4.595416e+10               500000000.0  4.595416e+10   \n",
              "1          6     4.595416e+10               500000000.0  4.140369e+10   \n",
              "2          6     4.595416e+10               500000000.0  4.600000e+10   \n",
              "3          6     4.595416e+10               500000000.0  4.595416e+10   \n",
              "4          6     4.595416e+10               500000000.0  6.848287e+10   \n",
              "5          6     4.595416e+10               500000000.0  4.218294e+10   \n",
              "6          6     4.595416e+10               500000000.0  4.054094e+10   \n",
              "7          6     4.595416e+10               500000000.0  4.595416e+10   \n",
              "8          6     4.595416e+10               500000000.0  4.600000e+10   \n",
              "9          6     4.595416e+10               500000000.0  4.157364e+10   \n",
              "\n",
              "   max_priority_fee  \n",
              "0      5.000000e+08  \n",
              "1      1.000000e+09  \n",
              "2      2.000000e+09  \n",
              "3      5.000000e+08  \n",
              "4      1.109231e+09  \n",
              "5      5.000000e+08  \n",
              "6      1.167582e+08  \n",
              "7      5.000000e+08  \n",
              "8      2.000000e+09  \n",
              "9      5.000000e+08  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d1acda46-feb4-4b03-9aa9-65fa8823357d\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>block_number</th>\n",
              "      <th>hash</th>\n",
              "      <th>index</th>\n",
              "      <th>gas_limit</th>\n",
              "      <th>gas_price</th>\n",
              "      <th>gas_used</th>\n",
              "      <th>max_fee_per_gas</th>\n",
              "      <th>max_priority_fee_per_gas</th>\n",
              "      <th>base_fee</th>\n",
              "      <th>status</th>\n",
              "      <th>type</th>\n",
              "      <th>data_tag</th>\n",
              "      <th>data_bytes</th>\n",
              "      <th>gas_bucket</th>\n",
              "      <th>max_fee_imputed</th>\n",
              "      <th>max_priority_fee_imputed</th>\n",
              "      <th>max_fee</th>\n",
              "      <th>max_priority_fee</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x971431be8deee130ff52febda56bdde09e3435e85d0f...</td>\n",
              "      <td>14</td>\n",
              "      <td>159222</td>\n",
              "      <td>34885439765</td>\n",
              "      <td>104935</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>&lt;NA&gt;</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>other</td>\n",
              "      <td>132.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0xb5560d0f250edd82a4abd54b8258bef625b014b8d18f...</td>\n",
              "      <td>57</td>\n",
              "      <td>124588</td>\n",
              "      <td>33301451293</td>\n",
              "      <td>119600</td>\n",
              "      <td>41403693704</td>\n",
              "      <td>1000000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.140369e+10</td>\n",
              "      <td>1.000000e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0xe292b6f7cef8c6abff30eba47996d4f6480191a96726...</td>\n",
              "      <td>36</td>\n",
              "      <td>116900</td>\n",
              "      <td>34301451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>46000000000</td>\n",
              "      <td>2000000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.600000e+10</td>\n",
              "      <td>2.000000e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x41964d9d770c6a8256e88ae66cd010a386c0251f889d...</td>\n",
              "      <td>106</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>45954155418</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x8fa3a033cd5b42cc2bf5ebba748fd60233f70ce2e60d...</td>\n",
              "      <td>47</td>\n",
              "      <td>200000</td>\n",
              "      <td>33410682710</td>\n",
              "      <td>105818</td>\n",
              "      <td>68482871585</td>\n",
              "      <td>1109231417</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>164.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>6.848287e+10</td>\n",
              "      <td>1.109231e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x998c815f1edd2b470629b6165cede9c55f756fc4e8ff...</td>\n",
              "      <td>113</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>42182943234</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.218294e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x5e9f1ab7d9f31b137b548f8d11b5366aed9c2e2cc48f...</td>\n",
              "      <td>132</td>\n",
              "      <td>116900</td>\n",
              "      <td>32418209449</td>\n",
              "      <td>110595</td>\n",
              "      <td>40540942256</td>\n",
              "      <td>116758156</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.054094e+10</td>\n",
              "      <td>1.167582e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x81b678cf78e276a3b6a144790ef270afaed5decfb899...</td>\n",
              "      <td>104</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>45954155418</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x3f7f7e3c5dea859d71d74d4cd5c4d5771d3f6e7200e7...</td>\n",
              "      <td>24</td>\n",
              "      <td>116900</td>\n",
              "      <td>34301451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>46000000000</td>\n",
              "      <td>2000000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.600000e+10</td>\n",
              "      <td>2.000000e+09</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>16694584</td>\n",
              "      <td>0x3b76d38e77f2c5a29e23fb0dc770094b7070d6e68243...</td>\n",
              "      <td>77</td>\n",
              "      <td>116900</td>\n",
              "      <td>32801451293</td>\n",
              "      <td>110595</td>\n",
              "      <td>41573640362</td>\n",
              "      <td>500000000</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>6</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>500000000.0</td>\n",
              "      <td>4.157364e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d1acda46-feb4-4b03-9aa9-65fa8823357d')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d1acda46-feb4-4b03-9aa9-65fa8823357d button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d1acda46-feb4-4b03-9aa9-65fa8823357d');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Get blob data"
      ],
      "metadata": {
        "id": "s8tKqjPI4phg"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "df['gas_used_calldata'] = df.apply(lambda x: 16*x['data_bytes'], axis=1)"
      ],
      "metadata": {
        "id": "k6Q73wJQMaCx"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dfo = df[['block_number', 'index', 'gas_used', 'max_fee', 'max_priority_fee', 'base_fee', 'data_tag', 'data_bytes', 'gas_used_calldata']].copy()\n",
        "dfo.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "qN1D8kCZOUKY",
        "outputId": "98eebb7a-2517-447a-c889-cab5c9a01f75"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   block_number  index  gas_used       max_fee  max_priority_fee     base_fee  \\\n",
              "0      16694584     14    104935  4.595416e+10      5.000000e+08  32301451293   \n",
              "1      16694584     57    119600  4.140369e+10      1.000000e+09  32301451293   \n",
              "2      16694584     36    110595  4.600000e+10      2.000000e+09  32301451293   \n",
              "3      16694584    106    110595  4.595416e+10      5.000000e+08  32301451293   \n",
              "4      16694584     47    105818  6.848287e+10      1.109231e+09  32301451293   \n",
              "\n",
              "  data_tag  data_bytes  gas_used_calldata  \n",
              "0    other       132.0             2112.0  \n",
              "1    other        36.0              576.0  \n",
              "2    other        36.0              576.0  \n",
              "3    other        36.0              576.0  \n",
              "4    other       164.0             2624.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2b05ca18-ba8b-4d0b-9d32-b6f6ce1f4ac8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>block_number</th>\n",
              "      <th>index</th>\n",
              "      <th>gas_used</th>\n",
              "      <th>max_fee</th>\n",
              "      <th>max_priority_fee</th>\n",
              "      <th>base_fee</th>\n",
              "      <th>data_tag</th>\n",
              "      <th>data_bytes</th>\n",
              "      <th>gas_used_calldata</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>16694584</td>\n",
              "      <td>14</td>\n",
              "      <td>104935</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>other</td>\n",
              "      <td>132.0</td>\n",
              "      <td>2112.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>16694584</td>\n",
              "      <td>57</td>\n",
              "      <td>119600</td>\n",
              "      <td>4.140369e+10</td>\n",
              "      <td>1.000000e+09</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>576.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16694584</td>\n",
              "      <td>36</td>\n",
              "      <td>110595</td>\n",
              "      <td>4.600000e+10</td>\n",
              "      <td>2.000000e+09</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>576.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>16694584</td>\n",
              "      <td>106</td>\n",
              "      <td>110595</td>\n",
              "      <td>4.595416e+10</td>\n",
              "      <td>5.000000e+08</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>other</td>\n",
              "      <td>36.0</td>\n",
              "      <td>576.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>16694584</td>\n",
              "      <td>47</td>\n",
              "      <td>105818</td>\n",
              "      <td>6.848287e+10</td>\n",
              "      <td>1.109231e+09</td>\n",
              "      <td>32301451293</td>\n",
              "      <td>other</td>\n",
              "      <td>164.0</td>\n",
              "      <td>2624.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2b05ca18-ba8b-4d0b-9d32-b6f6ce1f4ac8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2b05ca18-ba8b-4d0b-9d32-b6f6ce1f4ac8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2b05ca18-ba8b-4d0b-9d32-b6f6ce1f4ac8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dfo[dfo['data_tag']=='arb_data'].describe(percentiles=[0.5])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "Kx7ogWWAWI68",
        "outputId": "e5be92b6-9632-44ad-9ba2-77dd90d9879f"
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       block_number        index      gas_used       max_fee  \\\n",
              "count  1.055000e+03  1055.000000  1.055000e+03  1.055000e+03   \n",
              "mean   1.669811e+07   151.733649  1.842386e+06  5.774755e+10   \n",
              "std    1.817704e+03    57.029322  1.102684e+04  1.573698e+10   \n",
              "min    1.669452e+07     6.000000  1.673783e+06  4.224785e+10   \n",
              "50%    1.669827e+07   143.000000  1.844338e+06  5.613642e+10   \n",
              "max    1.670160e+07   655.000000  1.858159e+06  1.952515e+11   \n",
              "\n",
              "       max_priority_fee      base_fee     data_bytes  gas_used_calldata  \n",
              "count      1.055000e+03  1.055000e+03    1055.000000       1.055000e+03  \n",
              "mean       9.121099e+07  3.137498e+10   99275.552607       1.588409e+06  \n",
              "std        5.116870e+07  9.751770e+09     636.380071       1.018208e+04  \n",
              "min        6.141737e+07  2.281213e+10   89316.000000       1.429056e+06  \n",
              "50%        9.426717e+07  2.825990e+10   99396.000000       1.590336e+06  \n",
              "max        1.100000e+09  9.821040e+10  100164.000000       1.602624e+06  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c5c08d8e-115b-4158-ab5a-4db844f9d127\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>block_number</th>\n",
              "      <th>index</th>\n",
              "      <th>gas_used</th>\n",
              "      <th>max_fee</th>\n",
              "      <th>max_priority_fee</th>\n",
              "      <th>base_fee</th>\n",
              "      <th>data_bytes</th>\n",
              "      <th>gas_used_calldata</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>1.055000e+03</td>\n",
              "      <td>1055.000000</td>\n",
              "      <td>1.055000e+03</td>\n",
              "      <td>1.055000e+03</td>\n",
              "      <td>1.055000e+03</td>\n",
              "      <td>1.055000e+03</td>\n",
              "      <td>1055.000000</td>\n",
              "      <td>1.055000e+03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.669811e+07</td>\n",
              "      <td>151.733649</td>\n",
              "      <td>1.842386e+06</td>\n",
              "      <td>5.774755e+10</td>\n",
              "      <td>9.121099e+07</td>\n",
              "      <td>3.137498e+10</td>\n",
              "      <td>99275.552607</td>\n",
              "      <td>1.588409e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>1.817704e+03</td>\n",
              "      <td>57.029322</td>\n",
              "      <td>1.102684e+04</td>\n",
              "      <td>1.573698e+10</td>\n",
              "      <td>5.116870e+07</td>\n",
              "      <td>9.751770e+09</td>\n",
              "      <td>636.380071</td>\n",
              "      <td>1.018208e+04</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.669452e+07</td>\n",
              "      <td>6.000000</td>\n",
              "      <td>1.673783e+06</td>\n",
              "      <td>4.224785e+10</td>\n",
              "      <td>6.141737e+07</td>\n",
              "      <td>2.281213e+10</td>\n",
              "      <td>89316.000000</td>\n",
              "      <td>1.429056e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>1.669827e+07</td>\n",
              "      <td>143.000000</td>\n",
              "      <td>1.844338e+06</td>\n",
              "      <td>5.613642e+10</td>\n",
              "      <td>9.426717e+07</td>\n",
              "      <td>2.825990e+10</td>\n",
              "      <td>99396.000000</td>\n",
              "      <td>1.590336e+06</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.670160e+07</td>\n",
              "      <td>655.000000</td>\n",
              "      <td>1.858159e+06</td>\n",
              "      <td>1.952515e+11</td>\n",
              "      <td>1.100000e+09</td>\n",
              "      <td>9.821040e+10</td>\n",
              "      <td>100164.000000</td>\n",
              "      <td>1.602624e+06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c5c08d8e-115b-4158-ab5a-4db844f9d127')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c5c08d8e-115b-4158-ab5a-4db844f9d127 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c5c08d8e-115b-4158-ab5a-4db844f9d127');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dfo[dfo['data_tag']=='arb_data'].sort_values('block_number')['block_number'].drop_duplicates().diff().describe(percentiles=[0.5])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SWhN8mvYZFjI",
        "outputId": "7f7f5164-2d17-4e0f-a048-3233701609b3"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    1029.000000\n",
              "mean        6.874636\n",
              "std         2.791181\n",
              "min         1.000000\n",
              "50%         7.000000\n",
              "max        37.000000\n",
              "Name: block_number, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dfo[dfo['data_tag']=='op_data'].describe(percentiles=[0.5])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 238
        },
        "id": "LsQz2oaPWh9q",
        "outputId": "2a3dee92-d7bf-4565-a720-49af86d51747"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       block_number        index      gas_used       max_fee  \\\n",
              "count  2.982000e+03  2982.000000  2.982000e+03  2.982000e+03   \n",
              "mean   1.669793e+07   152.003018  5.981255e+05  6.091062e+10   \n",
              "std    2.045471e+03    54.047543  3.178673e+05  1.730274e+10   \n",
              "min    1.669452e+07     9.000000  2.595720e+05  4.136784e+10   \n",
              "50%    1.669782e+07   140.000000  5.031515e+05  5.634468e+10   \n",
              "max    1.670160e+07   745.000000  2.007252e+06  2.308838e+11   \n",
              "\n",
              "       max_priority_fee      base_fee     data_bytes  gas_used_calldata  \n",
              "count      2.982000e+03  2.982000e+03    2982.000000       2.982000e+03  \n",
              "mean       9.632323e+07  3.026605e+10   31071.493964       4.971439e+05  \n",
              "std        5.315175e+07  8.609011e+09   19897.477949       3.183596e+05  \n",
              "min        6.141737e+07  2.198410e+10   10008.000000       1.601280e+05  \n",
              "50%        1.000000e+08  2.798354e+10   25185.000000       4.029600e+05  \n",
              "max        1.000000e+09  1.116246e+11  119589.000000       1.913424e+06  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2234bf21-eb95-480e-b7b4-8cd46842c844\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>block_number</th>\n",
              "      <th>index</th>\n",
              "      <th>gas_used</th>\n",
              "      <th>max_fee</th>\n",
              "      <th>max_priority_fee</th>\n",
              "      <th>base_fee</th>\n",
              "      <th>data_bytes</th>\n",
              "      <th>gas_used_calldata</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>count</th>\n",
              "      <td>2.982000e+03</td>\n",
              "      <td>2982.000000</td>\n",
              "      <td>2.982000e+03</td>\n",
              "      <td>2.982000e+03</td>\n",
              "      <td>2.982000e+03</td>\n",
              "      <td>2.982000e+03</td>\n",
              "      <td>2982.000000</td>\n",
              "      <td>2.982000e+03</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>mean</th>\n",
              "      <td>1.669793e+07</td>\n",
              "      <td>152.003018</td>\n",
              "      <td>5.981255e+05</td>\n",
              "      <td>6.091062e+10</td>\n",
              "      <td>9.632323e+07</td>\n",
              "      <td>3.026605e+10</td>\n",
              "      <td>31071.493964</td>\n",
              "      <td>4.971439e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>std</th>\n",
              "      <td>2.045471e+03</td>\n",
              "      <td>54.047543</td>\n",
              "      <td>3.178673e+05</td>\n",
              "      <td>1.730274e+10</td>\n",
              "      <td>5.315175e+07</td>\n",
              "      <td>8.609011e+09</td>\n",
              "      <td>19897.477949</td>\n",
              "      <td>3.183596e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>min</th>\n",
              "      <td>1.669452e+07</td>\n",
              "      <td>9.000000</td>\n",
              "      <td>2.595720e+05</td>\n",
              "      <td>4.136784e+10</td>\n",
              "      <td>6.141737e+07</td>\n",
              "      <td>2.198410e+10</td>\n",
              "      <td>10008.000000</td>\n",
              "      <td>1.601280e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>50%</th>\n",
              "      <td>1.669782e+07</td>\n",
              "      <td>140.000000</td>\n",
              "      <td>5.031515e+05</td>\n",
              "      <td>5.634468e+10</td>\n",
              "      <td>1.000000e+08</td>\n",
              "      <td>2.798354e+10</td>\n",
              "      <td>25185.000000</td>\n",
              "      <td>4.029600e+05</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>max</th>\n",
              "      <td>1.670160e+07</td>\n",
              "      <td>745.000000</td>\n",
              "      <td>2.007252e+06</td>\n",
              "      <td>2.308838e+11</td>\n",
              "      <td>1.000000e+09</td>\n",
              "      <td>1.116246e+11</td>\n",
              "      <td>119589.000000</td>\n",
              "      <td>1.913424e+06</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2234bf21-eb95-480e-b7b4-8cd46842c844')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2234bf21-eb95-480e-b7b4-8cd46842c844 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2234bf21-eb95-480e-b7b4-8cd46842c844');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dfo[dfo['data_tag']=='op_data'].sort_values('block_number')['block_number'].drop_duplicates().diff().describe(percentiles=[0.5])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LAzmOAGbZomL",
        "outputId": "ed8747f6-bdac-40d6-ab0e-b4fe550cf8f9"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "count    2981.000000\n",
              "mean        2.375713\n",
              "std         0.778745\n",
              "min         2.000000\n",
              "50%         2.000000\n",
              "max        12.000000\n",
              "Name: block_number, dtype: float64"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dfo.to_csv('4844-sample.csv', index=False)"
      ],
      "metadata": {
        "id": "TDOsfHrzQwPt"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "9VrF2EjzwR0q"
      },
      "execution_count": 17,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "cell_execution_strategy": "setup",
      "provenance": [],
      "collapsed_sections": [
        "pj42soltXW_H"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}