diff --git a/your-code/.ipynb_checkpoints/main-checkpoint.ipynb b/your-code/.ipynb_checkpoints/main-checkpoint.ipynb index 31724c5..afa35d2 100644 --- a/your-code/.ipynb_checkpoints/main-checkpoint.ipynb +++ b/your-code/.ipynb_checkpoints/main-checkpoint.ipynb @@ -9,10 +9,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pandas as pd" + ] }, { "cell_type": "markdown", @@ -23,10 +25,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pymysql" + ] }, { "cell_type": "markdown", @@ -37,10 +41,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sqlalchemy import create_engine" + ] }, { "cell_type": "markdown", @@ -51,10 +57,208 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "
| \n", + " | id | \n", + "year | \n", + "month | \n", + "element | \n", + "d1 | \n", + "d2 | \n", + "d3 | \n", + "d4 | \n", + "d5 | \n", + "d6 | \n", + "... | \n", + "d22 | \n", + "d23 | \n", + "d24 | \n", + "d25 | \n", + "d26 | \n", + "d27 | \n", + "d28 | \n", + "d29 | \n", + "d30 | \n", + "d31 | \n", + "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "MX17004 | \n", + "2010 | \n", + "1 | \n", + "tmax | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "27.8 | \n", + "NaN | \n", + "
| 1 | \n", + "MX17004 | \n", + "2010 | \n", + "1 | \n", + "tmin | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "14.5 | \n", + "NaN | \n", + "
| 2 | \n", + "MX17004 | \n", + "2010 | \n", + "2 | \n", + "tmax | \n", + "NaN | \n", + "27.3 | \n", + "24.1 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "29.9 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
| 3 | \n", + "MX17004 | \n", + "2010 | \n", + "2 | \n", + "tmin | \n", + "NaN | \n", + "14.4 | \n", + "14.4 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "10.7 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
| 4 | \n", + "MX17004 | \n", + "2010 | \n", + "3 | \n", + "tmax | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "32.1 | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
5 rows × 35 columns
\n", + "| \n", + " | Id | \n", + "Reputation | \n", + "CreationDate | \n", + "DisplayName | \n", + "LastAccessDate | \n", + "WebsiteUrl | \n", + "Location | \n", + "AboutMe | \n", + "Views | \n", + "UpVotes | \n", + "DownVotes | \n", + "AccountId | \n", + "Age | \n", + "ProfileImageUrl | \n", + "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "-1 | \n", + "1 | \n", + "2010-07-19 06:55:26 | \n", + "Community | \n", + "2010-07-19 06:55:26 | \n", + "http://meta.stackexchange.com/ | \n", + "on the server farm | \n", + "<p>Hi, I'm not really a person.</p>\\n\\n<p>I'm ... | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "-1 | \n", + "NaN | \n", + "None | \n", + "
| 1 | \n", + "2 | \n", + "101 | \n", + "2010-07-19 14:01:36 | \n", + "Geoff Dalgas | \n", + "2013-11-12 22:07:23 | \n", + "http://stackoverflow.com | \n", + "Corvallis, OR | \n", + "<p>Developer on the StackOverflow team. Find ... | \n", + "25 | \n", + "3 | \n", + "0 | \n", + "2 | \n", + "37.0 | \n", + "None | \n", + "
| 2 | \n", + "3 | \n", + "101 | \n", + "2010-07-19 15:34:50 | \n", + "Jarrod Dixon | \n", + "2014-08-08 06:42:58 | \n", + "http://stackoverflow.com | \n", + "New York, NY | \n", + "<p><a href=\"http://blog.stackoverflow.com/2009... | \n", + "22 | \n", + "19 | \n", + "0 | \n", + "3 | \n", + "35.0 | \n", + "None | \n", + "
Hi, I'm not really a person.
\\n\\nI'm ... 0 5007 \n", + "1
Developer on the StackOverflow team. Find ... 25 3 \n", + "2
| \n", + " | userId | \n", + "Reputation | \n", + "Views | \n", + "UpVotes | \n", + "DownVotes | \n", + "postId | \n", + "Score | \n", + "ViewCount | \n", + "CommentCount | \n", + "
|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "2175 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 1 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8576 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 2 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8578 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 3 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8981 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 4 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8982 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| ... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "
| 90579 | \n", + "55734 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "115352 | \n", + "0 | \n", + "16.0 | \n", + "0 | \n", + "
| 90580 | \n", + "55738 | \n", + "11 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "115360 | \n", + "2 | \n", + "40.0 | \n", + "4 | \n", + "
| 90581 | \n", + "55742 | \n", + "6 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "115366 | \n", + "1 | \n", + "17.0 | \n", + "0 | \n", + "
| 90582 | \n", + "55744 | \n", + "6 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "115370 | \n", + "1 | \n", + "13.0 | \n", + "2 | \n", + "
| 90583 | \n", + "55746 | \n", + "106 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "115376 | \n", + "1 | \n", + "5.0 | \n", + "2 | \n", + "
90584 rows × 9 columns
\n", + "" + ], + "text/plain": [ + " userId Reputation Views UpVotes DownVotes postId Score \\\n", + "0 -1 1 0 5007 1920 2175 0 \n", + "1 -1 1 0 5007 1920 8576 0 \n", + "2 -1 1 0 5007 1920 8578 0 \n", + "3 -1 1 0 5007 1920 8981 0 \n", + "4 -1 1 0 5007 1920 8982 0 \n", + "... ... ... ... ... ... ... ... \n", + "90579 55734 1 0 0 0 115352 0 \n", + "90580 55738 11 0 0 0 115360 2 \n", + "90581 55742 6 0 0 0 115366 1 \n", + "90582 55744 6 1 0 0 115370 1 \n", + "90583 55746 106 1 0 0 115376 1 \n", + "\n", + " ViewCount CommentCount \n", + "0 NaN 0 \n", + "1 NaN 0 \n", + "2 NaN 0 \n", + "3 NaN 0 \n", + "4 NaN 0 \n", + "... ... ... \n", + "90579 16.0 0 \n", + "90580 40.0 4 \n", + "90581 17.0 0 \n", + "90582 13.0 2 \n", + "90583 5.0 2 \n", + "\n", + "[90584 rows x 9 columns]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union" + ] }, { "cell_type": "markdown", @@ -138,10 +763,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 0\n", + "Reputation 0\n", + "Views 0\n", + "UpVotes 0\n", + "DownVotes 0\n", + "postId 0\n", + "Score 0\n", + "ViewCount 48396\n", + "CommentCount 0\n", + "dtype: int64" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union.isnull().sum()\n", + "#There are 48,396 missing values in column 'ViewCount' " + ] }, { "cell_type": "markdown", @@ -153,10 +801,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 55746.0\n", + "Reputation 87393.0\n", + "Views 20932.0\n", + "UpVotes 11442.0\n", + "DownVotes 1920.0\n", + "postId 115378.0\n", + "Score 192.0\n", + "ViewCount 175495.0\n", + "CommentCount 45.0\n", + "dtype: float64" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I would not erase all rows with missing values in column 'ViewCount' because data of other \n", + "#columns are useful too. Instead I would replace it with zeros and I would notice in the analysis\n", + "#that involve column 'ViewCount' that important fact and the lack of realiability of its conclusions.\n", + "\n", + "Union['ViewCount']=Union['ViewCount'].fillna(0)\n", + "Union.max()" + ] }, { "cell_type": "markdown", @@ -167,10 +842,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId int64\n", + "Reputation int64\n", + "Views int64\n", + "UpVotes int64\n", + "DownVotes int64\n", + "postId int64\n", + "Score int64\n", + "ViewCount float64\n", + "CommentCount int64\n", + "dtype: object" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I think would be better that columns 'userId' and 'postId' were string data type because\n", + "#there won't be calculations on it and I would change the numerical type of the other columns from \n", + "#int64 or float to int32, I cannot see a number greater than 2,147,483,647 and it would be supposed \n", + "#increase speed processing.\n", + "Union.dtypes\n", + "to_string=['userId','postId']\n", + "to_int32=['Reputation','Views','UpVotes','DownVotes','Score','ViewCount','CommentCount']\n", + "Union[to_string].astype(str)\n", + "Union[to_int32].astype('int32')\n" + ] }, { "cell_type": "markdown", @@ -196,7 +901,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.4" } }, "nbformat": 4, diff --git a/your-code/main.ipynb b/your-code/main.ipynb index 31724c5..afa35d2 100644 --- a/your-code/main.ipynb +++ b/your-code/main.ipynb @@ -9,10 +9,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pandas as pd" + ] }, { "cell_type": "markdown", @@ -23,10 +25,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pymysql" + ] }, { "cell_type": "markdown", @@ -37,10 +41,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sqlalchemy import create_engine" + ] }, { "cell_type": "markdown", @@ -51,10 +57,208 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "| \n", + " | id | \n", + "year | \n", + "month | \n", + "element | \n", + "d1 | \n", + "d2 | \n", + "d3 | \n", + "d4 | \n", + "d5 | \n", + "d6 | \n", + "... | \n", + "d22 | \n", + "d23 | \n", + "d24 | \n", + "d25 | \n", + "d26 | \n", + "d27 | \n", + "d28 | \n", + "d29 | \n", + "d30 | \n", + "d31 | \n", + "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "MX17004 | \n", + "2010 | \n", + "1 | \n", + "tmax | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "27.8 | \n", + "NaN | \n", + "
| 1 | \n", + "MX17004 | \n", + "2010 | \n", + "1 | \n", + "tmin | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "14.5 | \n", + "NaN | \n", + "
| 2 | \n", + "MX17004 | \n", + "2010 | \n", + "2 | \n", + "tmax | \n", + "NaN | \n", + "27.3 | \n", + "24.1 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "29.9 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
| 3 | \n", + "MX17004 | \n", + "2010 | \n", + "2 | \n", + "tmin | \n", + "NaN | \n", + "14.4 | \n", + "14.4 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "10.7 | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
| 4 | \n", + "MX17004 | \n", + "2010 | \n", + "3 | \n", + "tmax | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "32.1 | \n", + "NaN | \n", + "... | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "NaN | \n", + "
5 rows × 35 columns
\n", + "| \n", + " | Id | \n", + "Reputation | \n", + "CreationDate | \n", + "DisplayName | \n", + "LastAccessDate | \n", + "WebsiteUrl | \n", + "Location | \n", + "AboutMe | \n", + "Views | \n", + "UpVotes | \n", + "DownVotes | \n", + "AccountId | \n", + "Age | \n", + "ProfileImageUrl | \n", + "
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "-1 | \n", + "1 | \n", + "2010-07-19 06:55:26 | \n", + "Community | \n", + "2010-07-19 06:55:26 | \n", + "http://meta.stackexchange.com/ | \n", + "on the server farm | \n", + "<p>Hi, I'm not really a person.</p>\\n\\n<p>I'm ... | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "-1 | \n", + "NaN | \n", + "None | \n", + "
| 1 | \n", + "2 | \n", + "101 | \n", + "2010-07-19 14:01:36 | \n", + "Geoff Dalgas | \n", + "2013-11-12 22:07:23 | \n", + "http://stackoverflow.com | \n", + "Corvallis, OR | \n", + "<p>Developer on the StackOverflow team. Find ... | \n", + "25 | \n", + "3 | \n", + "0 | \n", + "2 | \n", + "37.0 | \n", + "None | \n", + "
| 2 | \n", + "3 | \n", + "101 | \n", + "2010-07-19 15:34:50 | \n", + "Jarrod Dixon | \n", + "2014-08-08 06:42:58 | \n", + "http://stackoverflow.com | \n", + "New York, NY | \n", + "<p><a href=\"http://blog.stackoverflow.com/2009... | \n", + "22 | \n", + "19 | \n", + "0 | \n", + "3 | \n", + "35.0 | \n", + "None | \n", + "
Hi, I'm not really a person.
\\n\\nI'm ... 0 5007 \n", + "1
Developer on the StackOverflow team. Find ... 25 3 \n", + "2
| \n", + " | userId | \n", + "Reputation | \n", + "Views | \n", + "UpVotes | \n", + "DownVotes | \n", + "postId | \n", + "Score | \n", + "ViewCount | \n", + "CommentCount | \n", + "
|---|---|---|---|---|---|---|---|---|---|
| 0 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "2175 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 1 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8576 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 2 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8578 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 3 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8981 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| 4 | \n", + "-1 | \n", + "1 | \n", + "0 | \n", + "5007 | \n", + "1920 | \n", + "8982 | \n", + "0 | \n", + "NaN | \n", + "0 | \n", + "
| ... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "... | \n", + "
| 90579 | \n", + "55734 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "115352 | \n", + "0 | \n", + "16.0 | \n", + "0 | \n", + "
| 90580 | \n", + "55738 | \n", + "11 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "115360 | \n", + "2 | \n", + "40.0 | \n", + "4 | \n", + "
| 90581 | \n", + "55742 | \n", + "6 | \n", + "0 | \n", + "0 | \n", + "0 | \n", + "115366 | \n", + "1 | \n", + "17.0 | \n", + "0 | \n", + "
| 90582 | \n", + "55744 | \n", + "6 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "115370 | \n", + "1 | \n", + "13.0 | \n", + "2 | \n", + "
| 90583 | \n", + "55746 | \n", + "106 | \n", + "1 | \n", + "0 | \n", + "0 | \n", + "115376 | \n", + "1 | \n", + "5.0 | \n", + "2 | \n", + "
90584 rows × 9 columns
\n", + "" + ], + "text/plain": [ + " userId Reputation Views UpVotes DownVotes postId Score \\\n", + "0 -1 1 0 5007 1920 2175 0 \n", + "1 -1 1 0 5007 1920 8576 0 \n", + "2 -1 1 0 5007 1920 8578 0 \n", + "3 -1 1 0 5007 1920 8981 0 \n", + "4 -1 1 0 5007 1920 8982 0 \n", + "... ... ... ... ... ... ... ... \n", + "90579 55734 1 0 0 0 115352 0 \n", + "90580 55738 11 0 0 0 115360 2 \n", + "90581 55742 6 0 0 0 115366 1 \n", + "90582 55744 6 1 0 0 115370 1 \n", + "90583 55746 106 1 0 0 115376 1 \n", + "\n", + " ViewCount CommentCount \n", + "0 NaN 0 \n", + "1 NaN 0 \n", + "2 NaN 0 \n", + "3 NaN 0 \n", + "4 NaN 0 \n", + "... ... ... \n", + "90579 16.0 0 \n", + "90580 40.0 4 \n", + "90581 17.0 0 \n", + "90582 13.0 2 \n", + "90583 5.0 2 \n", + "\n", + "[90584 rows x 9 columns]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union" + ] }, { "cell_type": "markdown", @@ -138,10 +763,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 0\n", + "Reputation 0\n", + "Views 0\n", + "UpVotes 0\n", + "DownVotes 0\n", + "postId 0\n", + "Score 0\n", + "ViewCount 48396\n", + "CommentCount 0\n", + "dtype: int64" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union.isnull().sum()\n", + "#There are 48,396 missing values in column 'ViewCount' " + ] }, { "cell_type": "markdown", @@ -153,10 +801,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 55746.0\n", + "Reputation 87393.0\n", + "Views 20932.0\n", + "UpVotes 11442.0\n", + "DownVotes 1920.0\n", + "postId 115378.0\n", + "Score 192.0\n", + "ViewCount 175495.0\n", + "CommentCount 45.0\n", + "dtype: float64" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I would not erase all rows with missing values in column 'ViewCount' because data of other \n", + "#columns are useful too. Instead I would replace it with zeros and I would notice in the analysis\n", + "#that involve column 'ViewCount' that important fact and the lack of realiability of its conclusions.\n", + "\n", + "Union['ViewCount']=Union['ViewCount'].fillna(0)\n", + "Union.max()" + ] }, { "cell_type": "markdown", @@ -167,10 +842,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId int64\n", + "Reputation int64\n", + "Views int64\n", + "UpVotes int64\n", + "DownVotes int64\n", + "postId int64\n", + "Score int64\n", + "ViewCount float64\n", + "CommentCount int64\n", + "dtype: object" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I think would be better that columns 'userId' and 'postId' were string data type because\n", + "#there won't be calculations on it and I would change the numerical type of the other columns from \n", + "#int64 or float to int32, I cannot see a number greater than 2,147,483,647 and it would be supposed \n", + "#increase speed processing.\n", + "Union.dtypes\n", + "to_string=['userId','postId']\n", + "to_int32=['Reputation','Views','UpVotes','DownVotes','Score','ViewCount','CommentCount']\n", + "Union[to_string].astype(str)\n", + "Union[to_int32].astype('int32')\n" + ] }, { "cell_type": "markdown", @@ -196,7 +901,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.4" } }, "nbformat": 4,