diff --git a/your-code/.ipynb_checkpoints/main-checkpoint.ipynb b/your-code/.ipynb_checkpoints/main-checkpoint.ipynb index 31724c5..afa35d2 100644 --- a/your-code/.ipynb_checkpoints/main-checkpoint.ipynb +++ b/your-code/.ipynb_checkpoints/main-checkpoint.ipynb @@ -9,10 +9,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pandas as pd" + ] }, { "cell_type": "markdown", @@ -23,10 +25,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pymysql" + ] }, { "cell_type": "markdown", @@ -37,10 +41,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sqlalchemy import create_engine" + ] }, { "cell_type": "markdown", @@ -51,10 +57,208 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idyearmonthelementd1d2d3d4d5d6...d22d23d24d25d26d27d28d29d30d31
0MX1700420101tmaxNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN27.8NaN
1MX1700420101tminNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN14.5NaN
2MX1700420102tmaxNaN27.324.1NaNNaNNaN...NaN29.9NaNNaNNaNNaNNaNNaNNaNNaN
3MX1700420102tminNaN14.414.4NaNNaNNaN...NaN10.7NaNNaNNaNNaNNaNNaNNaNNaN
4MX1700420103tmaxNaNNaNNaNNaN32.1NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", + "

5 rows × 35 columns

\n", + "
" + ], + "text/plain": [ + " id year month element d1 d2 d3 d4 d5 d6 ... d22 d23 \\\n", + "0 MX17004 2010 1 tmax NaN NaN NaN NaN NaN NaN ... NaN NaN \n", + "1 MX17004 2010 1 tmin NaN NaN NaN NaN NaN NaN ... NaN NaN \n", + "2 MX17004 2010 2 tmax NaN 27.3 24.1 NaN NaN NaN ... NaN 29.9 \n", + "3 MX17004 2010 2 tmin NaN 14.4 14.4 NaN NaN NaN ... NaN 10.7 \n", + "4 MX17004 2010 3 tmax NaN NaN NaN NaN 32.1 NaN ... NaN NaN \n", + "\n", + " d24 d25 d26 d27 d28 d29 d30 d31 \n", + "0 NaN NaN NaN NaN NaN NaN 27.8 NaN \n", + "1 NaN NaN NaN NaN NaN NaN 14.5 NaN \n", + "2 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "3 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "4 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "\n", + "[5 rows x 35 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "engine = create_engine('mysql+pymysql://guest:relational@relational.fit.cvut.cz/stats')\n", + "df2 = pd.read_sql_table('users', con=engine) \n", + "#data=pd.read_csv(\"C:\\\\Users\\\\josea_000\\\\desktop\\\\data_labs\\\\lab-data-cleaning\\\\weather-raw.csv\")\n", + "#data.head(5)" + ] }, { "cell_type": "markdown", @@ -65,10 +269,60 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId object\n", + "year int64\n", + "month int64\n", + "element object\n", + "d1 float64\n", + "d2 float64\n", + "d3 float64\n", + "d4 float64\n", + "d5 float64\n", + "d6 float64\n", + "d7 float64\n", + "d8 float64\n", + "d9 float64\n", + "d10 float64\n", + "d11 float64\n", + "d12 float64\n", + "d13 float64\n", + "d14 float64\n", + "d15 float64\n", + "d16 float64\n", + "d17 float64\n", + "d18 float64\n", + "d19 float64\n", + "d20 float64\n", + "d21 float64\n", + "d22 float64\n", + "d23 float64\n", + "d24 float64\n", + "d25 float64\n", + "d26 float64\n", + "d27 float64\n", + "d28 float64\n", + "d29 float64\n", + "d30 float64\n", + "d31 float64\n", + "dtype: object" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.rename(columns={'id':'userId'}, inplace=True)\n", + "data.head(5)\n", + "data.dtypes" + ] }, { "cell_type": "markdown", @@ -79,10 +333,144 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "df = pd.read_sql_table('posts', con=engine)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IdReputationCreationDateDisplayNameLastAccessDateWebsiteUrlLocationAboutMeViewsUpVotesDownVotesAccountIdAgeProfileImageUrl
0-112010-07-19 06:55:26Community2010-07-19 06:55:26http://meta.stackexchange.com/on the server farm<p>Hi, I'm not really a person.</p>\\n\\n<p>I'm ...050071920-1NaNNone
121012010-07-19 14:01:36Geoff Dalgas2013-11-12 22:07:23http://stackoverflow.comCorvallis, OR<p>Developer on the StackOverflow team. Find ...2530237.0None
231012010-07-19 15:34:50Jarrod Dixon2014-08-08 06:42:58http://stackoverflow.comNew York, NY<p><a href=\"http://blog.stackoverflow.com/2009...22190335.0None
\n", + "
" + ], + "text/plain": [ + " Id Reputation CreationDate DisplayName LastAccessDate \\\n", + "0 -1 1 2010-07-19 06:55:26 Community 2010-07-19 06:55:26 \n", + "1 2 101 2010-07-19 14:01:36 Geoff Dalgas 2013-11-12 22:07:23 \n", + "2 3 101 2010-07-19 15:34:50 Jarrod Dixon 2014-08-08 06:42:58 \n", + "\n", + " WebsiteUrl Location \\\n", + "0 http://meta.stackexchange.com/ on the server farm \n", + "1 http://stackoverflow.com Corvallis, OR \n", + "2 http://stackoverflow.com New York, NY \n", + "\n", + " AboutMe Views UpVotes \\\n", + "0

Hi, I'm not really a person.

\\n\\n

I'm ... 0 5007 \n", + "1

Developer on the StackOverflow team. Find ... 25 3 \n", + "2

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdReputationViewsUpVotesDownVotespostIdScoreViewCountCommentCount
0-1105007192021750NaN0
1-1105007192085760NaN0
2-1105007192085780NaN0
3-1105007192089810NaN0
4-1105007192089820NaN0
..............................
90579557341000115352016.00
905805573811000115360240.04
90581557426000115366117.00
90582557446100115370113.02
905835574610610011537615.02
\n", + "

90584 rows × 9 columns

\n", + "" + ], + "text/plain": [ + " userId Reputation Views UpVotes DownVotes postId Score \\\n", + "0 -1 1 0 5007 1920 2175 0 \n", + "1 -1 1 0 5007 1920 8576 0 \n", + "2 -1 1 0 5007 1920 8578 0 \n", + "3 -1 1 0 5007 1920 8981 0 \n", + "4 -1 1 0 5007 1920 8982 0 \n", + "... ... ... ... ... ... ... ... \n", + "90579 55734 1 0 0 0 115352 0 \n", + "90580 55738 11 0 0 0 115360 2 \n", + "90581 55742 6 0 0 0 115366 1 \n", + "90582 55744 6 1 0 0 115370 1 \n", + "90583 55746 106 1 0 0 115376 1 \n", + "\n", + " ViewCount CommentCount \n", + "0 NaN 0 \n", + "1 NaN 0 \n", + "2 NaN 0 \n", + "3 NaN 0 \n", + "4 NaN 0 \n", + "... ... ... \n", + "90579 16.0 0 \n", + "90580 40.0 4 \n", + "90581 17.0 0 \n", + "90582 13.0 2 \n", + "90583 5.0 2 \n", + "\n", + "[90584 rows x 9 columns]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union" + ] }, { "cell_type": "markdown", @@ -138,10 +763,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 0\n", + "Reputation 0\n", + "Views 0\n", + "UpVotes 0\n", + "DownVotes 0\n", + "postId 0\n", + "Score 0\n", + "ViewCount 48396\n", + "CommentCount 0\n", + "dtype: int64" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union.isnull().sum()\n", + "#There are 48,396 missing values in column 'ViewCount' " + ] }, { "cell_type": "markdown", @@ -153,10 +801,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 55746.0\n", + "Reputation 87393.0\n", + "Views 20932.0\n", + "UpVotes 11442.0\n", + "DownVotes 1920.0\n", + "postId 115378.0\n", + "Score 192.0\n", + "ViewCount 175495.0\n", + "CommentCount 45.0\n", + "dtype: float64" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I would not erase all rows with missing values in column 'ViewCount' because data of other \n", + "#columns are useful too. Instead I would replace it with zeros and I would notice in the analysis\n", + "#that involve column 'ViewCount' that important fact and the lack of realiability of its conclusions.\n", + "\n", + "Union['ViewCount']=Union['ViewCount'].fillna(0)\n", + "Union.max()" + ] }, { "cell_type": "markdown", @@ -167,10 +842,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId int64\n", + "Reputation int64\n", + "Views int64\n", + "UpVotes int64\n", + "DownVotes int64\n", + "postId int64\n", + "Score int64\n", + "ViewCount float64\n", + "CommentCount int64\n", + "dtype: object" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I think would be better that columns 'userId' and 'postId' were string data type because\n", + "#there won't be calculations on it and I would change the numerical type of the other columns from \n", + "#int64 or float to int32, I cannot see a number greater than 2,147,483,647 and it would be supposed \n", + "#increase speed processing.\n", + "Union.dtypes\n", + "to_string=['userId','postId']\n", + "to_int32=['Reputation','Views','UpVotes','DownVotes','Score','ViewCount','CommentCount']\n", + "Union[to_string].astype(str)\n", + "Union[to_int32].astype('int32')\n" + ] }, { "cell_type": "markdown", @@ -196,7 +901,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.4" } }, "nbformat": 4, diff --git a/your-code/main.ipynb b/your-code/main.ipynb index 31724c5..afa35d2 100644 --- a/your-code/main.ipynb +++ b/your-code/main.ipynb @@ -9,10 +9,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pandas as pd" + ] }, { "cell_type": "markdown", @@ -23,10 +25,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pymysql" + ] }, { "cell_type": "markdown", @@ -37,10 +41,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sqlalchemy import create_engine" + ] }, { "cell_type": "markdown", @@ -51,10 +57,208 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idyearmonthelementd1d2d3d4d5d6...d22d23d24d25d26d27d28d29d30d31
0MX1700420101tmaxNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN27.8NaN
1MX1700420101tminNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaN14.5NaN
2MX1700420102tmaxNaN27.324.1NaNNaNNaN...NaN29.9NaNNaNNaNNaNNaNNaNNaNNaN
3MX1700420102tminNaN14.414.4NaNNaNNaN...NaN10.7NaNNaNNaNNaNNaNNaNNaNNaN
4MX1700420103tmaxNaNNaNNaNNaN32.1NaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", + "

5 rows × 35 columns

\n", + "
" + ], + "text/plain": [ + " id year month element d1 d2 d3 d4 d5 d6 ... d22 d23 \\\n", + "0 MX17004 2010 1 tmax NaN NaN NaN NaN NaN NaN ... NaN NaN \n", + "1 MX17004 2010 1 tmin NaN NaN NaN NaN NaN NaN ... NaN NaN \n", + "2 MX17004 2010 2 tmax NaN 27.3 24.1 NaN NaN NaN ... NaN 29.9 \n", + "3 MX17004 2010 2 tmin NaN 14.4 14.4 NaN NaN NaN ... NaN 10.7 \n", + "4 MX17004 2010 3 tmax NaN NaN NaN NaN 32.1 NaN ... NaN NaN \n", + "\n", + " d24 d25 d26 d27 d28 d29 d30 d31 \n", + "0 NaN NaN NaN NaN NaN NaN 27.8 NaN \n", + "1 NaN NaN NaN NaN NaN NaN 14.5 NaN \n", + "2 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "3 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "4 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "\n", + "[5 rows x 35 columns]" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "engine = create_engine('mysql+pymysql://guest:relational@relational.fit.cvut.cz/stats')\n", + "df2 = pd.read_sql_table('users', con=engine) \n", + "#data=pd.read_csv(\"C:\\\\Users\\\\josea_000\\\\desktop\\\\data_labs\\\\lab-data-cleaning\\\\weather-raw.csv\")\n", + "#data.head(5)" + ] }, { "cell_type": "markdown", @@ -65,10 +269,60 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId object\n", + "year int64\n", + "month int64\n", + "element object\n", + "d1 float64\n", + "d2 float64\n", + "d3 float64\n", + "d4 float64\n", + "d5 float64\n", + "d6 float64\n", + "d7 float64\n", + "d8 float64\n", + "d9 float64\n", + "d10 float64\n", + "d11 float64\n", + "d12 float64\n", + "d13 float64\n", + "d14 float64\n", + "d15 float64\n", + "d16 float64\n", + "d17 float64\n", + "d18 float64\n", + "d19 float64\n", + "d20 float64\n", + "d21 float64\n", + "d22 float64\n", + "d23 float64\n", + "d24 float64\n", + "d25 float64\n", + "d26 float64\n", + "d27 float64\n", + "d28 float64\n", + "d29 float64\n", + "d30 float64\n", + "d31 float64\n", + "dtype: object" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "data.rename(columns={'id':'userId'}, inplace=True)\n", + "data.head(5)\n", + "data.dtypes" + ] }, { "cell_type": "markdown", @@ -79,10 +333,144 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "df = pd.read_sql_table('posts', con=engine)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IdReputationCreationDateDisplayNameLastAccessDateWebsiteUrlLocationAboutMeViewsUpVotesDownVotesAccountIdAgeProfileImageUrl
0-112010-07-19 06:55:26Community2010-07-19 06:55:26http://meta.stackexchange.com/on the server farm<p>Hi, I'm not really a person.</p>\\n\\n<p>I'm ...050071920-1NaNNone
121012010-07-19 14:01:36Geoff Dalgas2013-11-12 22:07:23http://stackoverflow.comCorvallis, OR<p>Developer on the StackOverflow team. Find ...2530237.0None
231012010-07-19 15:34:50Jarrod Dixon2014-08-08 06:42:58http://stackoverflow.comNew York, NY<p><a href=\"http://blog.stackoverflow.com/2009...22190335.0None
\n", + "
" + ], + "text/plain": [ + " Id Reputation CreationDate DisplayName LastAccessDate \\\n", + "0 -1 1 2010-07-19 06:55:26 Community 2010-07-19 06:55:26 \n", + "1 2 101 2010-07-19 14:01:36 Geoff Dalgas 2013-11-12 22:07:23 \n", + "2 3 101 2010-07-19 15:34:50 Jarrod Dixon 2014-08-08 06:42:58 \n", + "\n", + " WebsiteUrl Location \\\n", + "0 http://meta.stackexchange.com/ on the server farm \n", + "1 http://stackoverflow.com Corvallis, OR \n", + "2 http://stackoverflow.com New York, NY \n", + "\n", + " AboutMe Views UpVotes \\\n", + "0

Hi, I'm not really a person.

\\n\\n

I'm ... 0 5007 \n", + "1

Developer on the StackOverflow team. Find ... 25 3 \n", + "2

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdReputationViewsUpVotesDownVotespostIdScoreViewCountCommentCount
0-1105007192021750NaN0
1-1105007192085760NaN0
2-1105007192085780NaN0
3-1105007192089810NaN0
4-1105007192089820NaN0
..............................
90579557341000115352016.00
905805573811000115360240.04
90581557426000115366117.00
90582557446100115370113.02
905835574610610011537615.02
\n", + "

90584 rows × 9 columns

\n", + "" + ], + "text/plain": [ + " userId Reputation Views UpVotes DownVotes postId Score \\\n", + "0 -1 1 0 5007 1920 2175 0 \n", + "1 -1 1 0 5007 1920 8576 0 \n", + "2 -1 1 0 5007 1920 8578 0 \n", + "3 -1 1 0 5007 1920 8981 0 \n", + "4 -1 1 0 5007 1920 8982 0 \n", + "... ... ... ... ... ... ... ... \n", + "90579 55734 1 0 0 0 115352 0 \n", + "90580 55738 11 0 0 0 115360 2 \n", + "90581 55742 6 0 0 0 115366 1 \n", + "90582 55744 6 1 0 0 115370 1 \n", + "90583 55746 106 1 0 0 115376 1 \n", + "\n", + " ViewCount CommentCount \n", + "0 NaN 0 \n", + "1 NaN 0 \n", + "2 NaN 0 \n", + "3 NaN 0 \n", + "4 NaN 0 \n", + "... ... ... \n", + "90579 16.0 0 \n", + "90580 40.0 4 \n", + "90581 17.0 0 \n", + "90582 13.0 2 \n", + "90583 5.0 2 \n", + "\n", + "[90584 rows x 9 columns]" + ] + }, + "execution_count": 28, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union" + ] }, { "cell_type": "markdown", @@ -138,10 +763,33 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 27, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 0\n", + "Reputation 0\n", + "Views 0\n", + "UpVotes 0\n", + "DownVotes 0\n", + "postId 0\n", + "Score 0\n", + "ViewCount 48396\n", + "CommentCount 0\n", + "dtype: int64" + ] + }, + "execution_count": 27, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "Union.isnull().sum()\n", + "#There are 48,396 missing values in column 'ViewCount' " + ] }, { "cell_type": "markdown", @@ -153,10 +801,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 32, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 55746.0\n", + "Reputation 87393.0\n", + "Views 20932.0\n", + "UpVotes 11442.0\n", + "DownVotes 1920.0\n", + "postId 115378.0\n", + "Score 192.0\n", + "ViewCount 175495.0\n", + "CommentCount 45.0\n", + "dtype: float64" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I would not erase all rows with missing values in column 'ViewCount' because data of other \n", + "#columns are useful too. Instead I would replace it with zeros and I would notice in the analysis\n", + "#that involve column 'ViewCount' that important fact and the lack of realiability of its conclusions.\n", + "\n", + "Union['ViewCount']=Union['ViewCount'].fillna(0)\n", + "Union.max()" + ] }, { "cell_type": "markdown", @@ -167,10 +842,40 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 42, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId int64\n", + "Reputation int64\n", + "Views int64\n", + "UpVotes int64\n", + "DownVotes int64\n", + "postId int64\n", + "Score int64\n", + "ViewCount float64\n", + "CommentCount int64\n", + "dtype: object" + ] + }, + "execution_count": 42, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "#I think would be better that columns 'userId' and 'postId' were string data type because\n", + "#there won't be calculations on it and I would change the numerical type of the other columns from \n", + "#int64 or float to int32, I cannot see a number greater than 2,147,483,647 and it would be supposed \n", + "#increase speed processing.\n", + "Union.dtypes\n", + "to_string=['userId','postId']\n", + "to_int32=['Reputation','Views','UpVotes','DownVotes','Score','ViewCount','CommentCount']\n", + "Union[to_string].astype(str)\n", + "Union[to_int32].astype('int32')\n" + ] }, { "cell_type": "markdown", @@ -196,7 +901,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.7.4" } }, "nbformat": 4,