diff --git a/your-code/main.ipynb b/your-code/main.ipynb index 31724c5..302694d 100644 --- a/your-code/main.ipynb +++ b/your-code/main.ipynb @@ -9,10 +9,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import pandas as pd\n", + "import cryptography" + ] }, { "cell_type": "markdown", @@ -23,24 +26,41 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from sqlalchemy import create_engine" + ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### 3. Create a mysql engine to set the connection to the server. Check the connection details in [this link](https://relational.fit.cvut.cz/search?tableCount%5B%5D=0-10&tableCount%5B%5D=10-30&dataType%5B%5D=Numeric&databaseSize%5B%5D=KB&databaseSize%5B%5D=MB)" + "#### 3. Create a mysql engine to set the connection to the server. Check the connection details in [this link](https://relational.fit.cvut.cz/dataset/Stats)" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "engine = create_engine(\"mysql+pymysql://guest:relational@relational.fit.cvut.cz:3306/stats\")\n", + "con = engine.connect()\n", + "con" + ] }, { "cell_type": "markdown", @@ -51,10 +71,15 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "users = pd.read_sql_query(\n", + " \"SELECT * FROM stats.users\",\n", + " engine\n", + ")" + ] }, { "cell_type": "markdown", @@ -65,10 +90,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "users = users.rename(columns = {\"Id\" : \"userId\"})" + ] }, { "cell_type": "markdown", @@ -79,10 +106,15 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "posts = pd.read_sql_query(\n", + " \"SELECT * FROM stats.posts\",\n", + " engine\n", + ")" + ] }, { "cell_type": "markdown", @@ -93,10 +125,152 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "posts = posts.rename(columns = {\"Id\" : \"postId\", \"OwnerUserId\" : \"userId\"})" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
postIdPostTypeIdAcceptedAnswerIdCreaionDateScoreViewCountBodyuserIdLasActivityDateTitle...AnswerCountCommentCountFavoriteCountLastEditorUserIdLastEditDateCommunityOwnedDateParentIdClosedDateOwnerDisplayNameLastEditorDisplayName
01115.02010-07-19 19:12:12231278.0<p>How should I elicit prior distributions fro...8.02010-09-15 21:08:26Eliciting priors from experts...5.0114.0NaNNaTNaTNaNNaTNoneNone
12159.02010-07-19 19:12:57228198.0<p>In many different statistical methods there...24.02012-11-12 09:21:54What is normality?...7.018.088.02010-08-07 17:56:44NaTNaNNaTNoneNone
\n", + "

2 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " postId PostTypeId AcceptedAnswerId CreaionDate Score ViewCount \\\n", + "0 1 1 15.0 2010-07-19 19:12:12 23 1278.0 \n", + "1 2 1 59.0 2010-07-19 19:12:57 22 8198.0 \n", + "\n", + " Body userId \\\n", + "0

How should I elicit prior distributions fro... 8.0 \n", + "1

In many different statistical methods there... 24.0 \n", + "\n", + " LasActivityDate Title ... AnswerCount \\\n", + "0 2010-09-15 21:08:26 Eliciting priors from experts ... 5.0 \n", + "1 2012-11-12 09:21:54 What is normality? ... 7.0 \n", + "\n", + " CommentCount FavoriteCount LastEditorUserId LastEditDate \\\n", + "0 1 14.0 NaN NaT \n", + "1 1 8.0 88.0 2010-08-07 17:56:44 \n", + "\n", + " CommunityOwnedDate ParentId ClosedDate OwnerDisplayName \\\n", + "0 NaT NaN NaT None \n", + "1 NaT NaN NaT None \n", + "\n", + " LastEditorDisplayName \n", + "0 None \n", + "1 None \n", + "\n", + "[2 rows x 21 columns]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "posts.head(2)" + ] }, { "cell_type": "markdown", @@ -109,10 +283,13 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "users_1 = users[[\"userId\", \"Reputation\", \"Views\", \"UpVotes\", \"DownVotes\"]]\n", + "posts_1 = posts[[\"postId\", \"Score\", \"userId\", \"ViewCount\", \"CommentCount\"]]" + ] }, { "cell_type": "markdown", @@ -124,10 +301,139 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "u_p_merge = pd.merge(left = users_1, right = posts_1, left_on = 'userId', right_on = \"userId\")" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "

\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdReputationViewsUpVotesDownVotespostIdScoreViewCountCommentCount
0-110500719202175000
1-110500719208576000
2-110500719208578000
3-110500719208981000
4-110500719208982000
\n", + "
" + ], + "text/plain": [ + " userId Reputation Views UpVotes DownVotes postId Score ViewCount \\\n", + "0 -1 1 0 5007 1920 2175 0 0 \n", + "1 -1 1 0 5007 1920 8576 0 0 \n", + "2 -1 1 0 5007 1920 8578 0 0 \n", + "3 -1 1 0 5007 1920 8981 0 0 \n", + "4 -1 1 0 5007 1920 8982 0 0 \n", + "\n", + " CommentCount \n", + "0 0 \n", + "1 0 \n", + "2 0 \n", + "3 0 \n", + "4 0 " + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "u_p_merge.head()" + ] }, { "cell_type": "markdown", @@ -138,10 +444,64 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "data": { + "text/plain": [ + "userId 0\n", + "Reputation 0\n", + "Views 0\n", + "UpVotes 0\n", + "DownVotes 0\n", + "postId 0\n", + "Score 0\n", + "ViewCount 48396\n", + "CommentCount 0\n", + "dtype: int64" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "u_p_merge.isnull().sum()" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Int64Index: 90584 entries, 0 to 90583\n", + "Data columns (total 9 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 userId 90584 non-null int64 \n", + " 1 Reputation 90584 non-null int64 \n", + " 2 Views 90584 non-null int64 \n", + " 3 UpVotes 90584 non-null int64 \n", + " 4 DownVotes 90584 non-null int64 \n", + " 5 postId 90584 non-null int64 \n", + " 6 Score 90584 non-null int64 \n", + " 7 ViewCount 42188 non-null float64\n", + " 8 CommentCount 90584 non-null int64 \n", + "dtypes: float64(1), int64(8)\n", + "memory usage: 6.9 MB\n" + ] + } + ], + "source": [ + "u_p_merge.info()" + ] }, { "cell_type": "markdown", @@ -151,12 +511,45 @@ "**Remember** to check the results of your code before passing to the next step" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "I will fill the values to keep the column structure logical and useful for analysis" + ] + }, { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, - "outputs": [], - "source": [] + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Int64Index: 90584 entries, 0 to 90583\n", + "Data columns (total 9 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 userId 90584 non-null int64 \n", + " 1 Reputation 90584 non-null int64 \n", + " 2 Views 90584 non-null int64 \n", + " 3 UpVotes 90584 non-null int64 \n", + " 4 DownVotes 90584 non-null int64 \n", + " 5 postId 90584 non-null int64 \n", + " 6 Score 90584 non-null int64 \n", + " 7 ViewCount 90584 non-null float64\n", + " 8 CommentCount 90584 non-null int64 \n", + "dtypes: float64(1), int64(8)\n", + "memory usage: 6.9 MB\n" + ] + } + ], + "source": [ + "u_p_merge[[\"ViewCount\"]] = u_p_merge[[\"ViewCount\"]].fillna(0)\n", + "u_p_merge.info()" + ] }, { "cell_type": "markdown", @@ -170,7 +563,41 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "u_p_merge[\"ViewCount\"] = u_p_merge[\"ViewCount\"].values.astype('int64')" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Int64Index: 90584 entries, 0 to 90583\n", + "Data columns (total 9 columns):\n", + " # Column Non-Null Count Dtype\n", + "--- ------ -------------- -----\n", + " 0 userId 90584 non-null int64\n", + " 1 Reputation 90584 non-null int64\n", + " 2 Views 90584 non-null int64\n", + " 3 UpVotes 90584 non-null int64\n", + " 4 DownVotes 90584 non-null int64\n", + " 5 postId 90584 non-null int64\n", + " 6 Score 90584 non-null int64\n", + " 7 ViewCount 90584 non-null int64\n", + " 8 CommentCount 90584 non-null int64\n", + "dtypes: int64(9)\n", + "memory usage: 6.9 MB\n" + ] + } + ], + "source": [ + "u_p_merge.info()" + ] }, { "cell_type": "markdown", @@ -196,9 +623,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.5" + "version": "3.8.5" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }