From fb457346eea0bd4717abe254b6b24cd1a72957b2 Mon Sep 17 00:00:00 2001 From: JulietteGoardon <142164143+JulietteGoardon@users.noreply.github.com> Date: Sat, 21 Oct 2023 18:00:46 +0200 Subject: [PATCH] Created using Colaboratory --- main.ipynb | 4231 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 4231 insertions(+) create mode 100644 main.ipynb diff --git a/main.ipynb b/main.ipynb new file mode 100644 index 0000000..326f084 --- /dev/null +++ b/main.ipynb @@ -0,0 +1,4231 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "view-in-github", + "colab_type": "text" + }, + "source": [ + "\"Open" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kAf8tbjdqtHZ" + }, + "source": [ + "#### 1. Import pandas library" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3nP0GpuqqtHa" + }, + "outputs": [], + "source": [ + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZfmWEifUqtHb" + }, + "source": [ + "#### 2. Import users table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 652 + }, + "id": "kD_xv1FXqtHb", + "outputId": "520042be-b7b2-4093-9690-b9b9926d5f39" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " Id Reputation CreationDate DisplayName \\\n", + "0 -1 1 2010-07-19 06:55:26 Community \n", + "1 2 101 2010-07-19 14:01:36 Geoff Dalgas \n", + "2 3 101 2010-07-19 15:34:50 Jarrod Dixon \n", + "3 4 101 2010-07-19 19:03:27 Emmett \n", + "4 5 6792 2010-07-19 19:03:57 Shane \n", + "... ... ... ... ... \n", + "40320 55743 1 2014-09-13 21:03:50 AussieMeg \n", + "40321 55744 6 2014-09-13 21:39:30 Mia Maria \n", + "40322 55745 101 2014-09-13 23:45:27 tronbabylove \n", + "40323 55746 106 2014-09-14 00:29:41 GPP \n", + "40324 55747 1 2014-09-14 01:01:44 Shivam Agrawal \n", + "\n", + " LastAccessDate WebsiteUrl \\\n", + "0 2010-07-19 06:55:26 http://meta.stackexchange.com/ \n", + "1 2013-11-12 22:07:23 http://stackoverflow.com \n", + "2 2014-08-08 06:42:58 http://stackoverflow.com \n", + "3 2014-01-02 09:31:02 http://minesweeperonline.com \n", + "4 2014-08-13 00:23:47 http://www.statalgo.com \n", + "... ... ... \n", + "40320 2014-09-13 21:18:52 NaN \n", + "40321 2014-09-13 21:39:30 NaN \n", + "40322 2014-09-13 23:45:27 NaN \n", + "40323 2014-09-14 02:05:17 NaN \n", + "40324 2014-09-14 01:19:04 NaN \n", + "\n", + " Location AboutMe \\\n", + "0 on the server farm

Hi, I'm not really a person.

\\r\\n\\r\\n

... \n", + "1 Corvallis, OR

Developer on the StackOverflow team. Find ... \n", + "2 New York, NY

currently at a startup in SF

\\r\\n\\r\\n

... \n", + "4 New York, NY

Quantitative researcher focusing on statist... \n", + "... ... ... \n", + "40320 NaN NaN \n", + "40321 NaN NaN \n", + "40322 United States NaN \n", + "40323 NaN

Stats noobie, product, marketing & medi... \n", + "40324 India

Maths Enthusiast

\\r\\n \n", + "\n", + " Views UpVotes DownVotes AccountId Age \\\n", + "0 0 5007 1920 -1 NaN \n", + "1 25 3 0 2 37.0 \n", + "2 22 19 0 3 35.0 \n", + "3 11 0 0 1998 28.0 \n", + "4 1145 662 5 54503 35.0 \n", + "... ... ... ... ... ... \n", + "40320 0 0 0 5026902 NaN \n", + "40321 1 0 0 5026998 NaN \n", + "40322 0 0 0 481766 NaN \n", + "40323 1 0 0 976289 NaN \n", + "40324 0 0 0 5027354 NaN \n", + "\n", + " ProfileImageUrl \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 http://i.stack.imgur.com/d1oHX.jpg \n", + "4 NaN \n", + "... ... \n", + "40320 http://graph.facebook.com/665821703/picture?ty... \n", + "40321 NaN \n", + "40322 https://www.gravatar.com/avatar/faa7a3fdbd8308... \n", + "40323 https://www.gravatar.com/avatar/6d9e9fa6b783a3... \n", + "40324 https://lh4.googleusercontent.com/-ZsXhwVaFmiY... \n", + "\n", + "[40325 rows x 14 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IdReputationCreationDateDisplayNameLastAccessDateWebsiteUrlLocationAboutMeViewsUpVotesDownVotesAccountIdAgeProfileImageUrl
0-112010-07-19 06:55:26Community2010-07-19 06:55:26http://meta.stackexchange.com/on the server farm<p>Hi, I'm not really a person.</p>\\r\\n\\r\\n<p>...050071920-1NaNNaN
121012010-07-19 14:01:36Geoff Dalgas2013-11-12 22:07:23http://stackoverflow.comCorvallis, OR<p>Developer on the StackOverflow team. Find ...2530237.0NaN
231012010-07-19 15:34:50Jarrod Dixon2014-08-08 06:42:58http://stackoverflow.comNew York, NY<p><a href=\"http://blog.stackoverflow.com/2009...22190335.0NaN
341012010-07-19 19:03:27Emmett2014-01-02 09:31:02http://minesweeperonline.comSan Francisco, CA<p>currently at a startup in SF</p>\\r\\n\\r\\n<p>...1100199828.0http://i.stack.imgur.com/d1oHX.jpg
4567922010-07-19 19:03:57Shane2014-08-13 00:23:47http://www.statalgo.comNew York, NY<p>Quantitative researcher focusing on statist...114566255450335.0NaN
.............................................
403205574312014-09-13 21:03:50AussieMeg2014-09-13 21:18:52NaNNaNNaN0005026902NaNhttp://graph.facebook.com/665821703/picture?ty...
403215574462014-09-13 21:39:30Mia Maria2014-09-13 21:39:30NaNNaNNaN1005026998NaNNaN
40322557451012014-09-13 23:45:27tronbabylove2014-09-13 23:45:27NaNUnited StatesNaN000481766NaNhttps://www.gravatar.com/avatar/faa7a3fdbd8308...
40323557461062014-09-14 00:29:41GPP2014-09-14 02:05:17NaNNaN<p>Stats noobie, product, marketing &amp; medi...100976289NaNhttps://www.gravatar.com/avatar/6d9e9fa6b783a3...
403245574712014-09-14 01:01:44Shivam Agrawal2014-09-14 01:19:04NaNIndia<p>Maths Enthusiast </p>\\r\\n0005027354NaNhttps://lh4.googleusercontent.com/-ZsXhwVaFmiY...
\n", + "

40325 rows × 14 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "execution_count": 4 + } + ], + "source": [ + "Users = pd.read_csv(\"users_table.csv\")\n", + "Users" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TRlVPpWzqtHb" + }, + "source": [ + "#### 3. Rename Id column to userId" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 652 + }, + "id": "JkzjOTJNqtHb", + "outputId": "8dda6df0-ff28-401d-821c-1c7dfeec263c" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " userId Reputation CreationDate DisplayName \\\n", + "0 -1 1 2010-07-19 06:55:26 Community \n", + "1 2 101 2010-07-19 14:01:36 Geoff Dalgas \n", + "2 3 101 2010-07-19 15:34:50 Jarrod Dixon \n", + "3 4 101 2010-07-19 19:03:27 Emmett \n", + "4 5 6792 2010-07-19 19:03:57 Shane \n", + "... ... ... ... ... \n", + "40320 55743 1 2014-09-13 21:03:50 AussieMeg \n", + "40321 55744 6 2014-09-13 21:39:30 Mia Maria \n", + "40322 55745 101 2014-09-13 23:45:27 tronbabylove \n", + "40323 55746 106 2014-09-14 00:29:41 GPP \n", + "40324 55747 1 2014-09-14 01:01:44 Shivam Agrawal \n", + "\n", + " LastAccessDate WebsiteUrl \\\n", + "0 2010-07-19 06:55:26 http://meta.stackexchange.com/ \n", + "1 2013-11-12 22:07:23 http://stackoverflow.com \n", + "2 2014-08-08 06:42:58 http://stackoverflow.com \n", + "3 2014-01-02 09:31:02 http://minesweeperonline.com \n", + "4 2014-08-13 00:23:47 http://www.statalgo.com \n", + "... ... ... \n", + "40320 2014-09-13 21:18:52 NaN \n", + "40321 2014-09-13 21:39:30 NaN \n", + "40322 2014-09-13 23:45:27 NaN \n", + "40323 2014-09-14 02:05:17 NaN \n", + "40324 2014-09-14 01:19:04 NaN \n", + "\n", + " Location AboutMe \\\n", + "0 on the server farm

Hi, I'm not really a person.

\\r\\n\\r\\n

... \n", + "1 Corvallis, OR

Developer on the StackOverflow team. Find ... \n", + "2 New York, NY

currently at a startup in SF

\\r\\n\\r\\n

... \n", + "4 New York, NY

Quantitative researcher focusing on statist... \n", + "... ... ... \n", + "40320 NaN NaN \n", + "40321 NaN NaN \n", + "40322 United States NaN \n", + "40323 NaN

Stats noobie, product, marketing & medi... \n", + "40324 India

Maths Enthusiast

\\r\\n \n", + "\n", + " Views UpVotes DownVotes AccountId Age \\\n", + "0 0 5007 1920 -1 NaN \n", + "1 25 3 0 2 37.0 \n", + "2 22 19 0 3 35.0 \n", + "3 11 0 0 1998 28.0 \n", + "4 1145 662 5 54503 35.0 \n", + "... ... ... ... ... ... \n", + "40320 0 0 0 5026902 NaN \n", + "40321 1 0 0 5026998 NaN \n", + "40322 0 0 0 481766 NaN \n", + "40323 1 0 0 976289 NaN \n", + "40324 0 0 0 5027354 NaN \n", + "\n", + " ProfileImageUrl \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 http://i.stack.imgur.com/d1oHX.jpg \n", + "4 NaN \n", + "... ... \n", + "40320 http://graph.facebook.com/665821703/picture?ty... \n", + "40321 NaN \n", + "40322 https://www.gravatar.com/avatar/faa7a3fdbd8308... \n", + "40323 https://www.gravatar.com/avatar/6d9e9fa6b783a3... \n", + "40324 https://lh4.googleusercontent.com/-ZsXhwVaFmiY... \n", + "\n", + "[40325 rows x 14 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdReputationCreationDateDisplayNameLastAccessDateWebsiteUrlLocationAboutMeViewsUpVotesDownVotesAccountIdAgeProfileImageUrl
0-112010-07-19 06:55:26Community2010-07-19 06:55:26http://meta.stackexchange.com/on the server farm<p>Hi, I'm not really a person.</p>\\r\\n\\r\\n<p>...050071920-1NaNNaN
121012010-07-19 14:01:36Geoff Dalgas2013-11-12 22:07:23http://stackoverflow.comCorvallis, OR<p>Developer on the StackOverflow team. Find ...2530237.0NaN
231012010-07-19 15:34:50Jarrod Dixon2014-08-08 06:42:58http://stackoverflow.comNew York, NY<p><a href=\"http://blog.stackoverflow.com/2009...22190335.0NaN
341012010-07-19 19:03:27Emmett2014-01-02 09:31:02http://minesweeperonline.comSan Francisco, CA<p>currently at a startup in SF</p>\\r\\n\\r\\n<p>...1100199828.0http://i.stack.imgur.com/d1oHX.jpg
4567922010-07-19 19:03:57Shane2014-08-13 00:23:47http://www.statalgo.comNew York, NY<p>Quantitative researcher focusing on statist...114566255450335.0NaN
.............................................
403205574312014-09-13 21:03:50AussieMeg2014-09-13 21:18:52NaNNaNNaN0005026902NaNhttp://graph.facebook.com/665821703/picture?ty...
403215574462014-09-13 21:39:30Mia Maria2014-09-13 21:39:30NaNNaNNaN1005026998NaNNaN
40322557451012014-09-13 23:45:27tronbabylove2014-09-13 23:45:27NaNUnited StatesNaN000481766NaNhttps://www.gravatar.com/avatar/faa7a3fdbd8308...
40323557461062014-09-14 00:29:41GPP2014-09-14 02:05:17NaNNaN<p>Stats noobie, product, marketing &amp; medi...100976289NaNhttps://www.gravatar.com/avatar/6d9e9fa6b783a3...
403245574712014-09-14 01:01:44Shivam Agrawal2014-09-14 01:19:04NaNIndia<p>Maths Enthusiast </p>\\r\\n0005027354NaNhttps://lh4.googleusercontent.com/-ZsXhwVaFmiY...
\n", + "

40325 rows × 14 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ], + "source": [ + "Users = Users.rename(columns={\"Id\": \"userId\"})\n", + "\n", + "Users\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kQpDwkonqtHb" + }, + "source": [ + "#### 4. Import posts table:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 843 + }, + "id": "t0DBMa-HqtHb", + "outputId": "eed59f32-da51-4281-a3f1-3d4d73002005" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " Id PostTypeId AcceptedAnswerId CreaionDate Score \\\n", + "0 1 1 15.0 2010-07-19 19:12:12 23 \n", + "1 2 1 59.0 2010-07-19 19:12:57 22 \n", + "2 3 1 5.0 2010-07-19 19:13:28 54 \n", + "3 4 1 135.0 2010-07-19 19:13:31 13 \n", + "4 5 2 NaN 2010-07-19 19:14:43 81 \n", + "... ... ... ... ... ... \n", + "39995 48321 2 NaN 2013-01-23 09:00:01 0 \n", + "39996 48322 2 NaN 2013-01-23 09:09:34 3 \n", + "39997 48323 2 NaN 2013-01-23 09:16:44 1 \n", + "39998 48324 2 NaN 2013-01-23 09:36:07 3 \n", + "39999 48325 1 NaN 2013-01-23 09:44:07 -1 \n", + "\n", + " ViewCount Body \\\n", + "0 1278.0

How should I elicit prior distributions fro... \n", + "1 8198.0

In many different statistical methods there... \n", + "2 3613.0

What are some valuable Statistical Analysis... \n", + "3 5224.0

I have two groups of data. Each with a dif... \n", + "4 NaN

The R-project

\\n\\n

you can use the matlab codes for svm and co... \n", + "39996 NaN

I use If I understand your question correctly, yo... \n", + "39998 NaN

Doesn't really help you with your question,... \n", + "39999 116.0

I have 10 vectors each having 100,000 point... \n", + "\n", + " OwnerUserId LasActivityDate \\\n", + "0 8.0 2010-09-15 21:08:26 \n", + "1 24.0 2012-11-12 09:21:54 \n", + "2 18.0 2013-05-27 14:48:36 \n", + "3 23.0 2010-09-08 03:00:19 \n", + "4 23.0 2010-07-19 19:21:15 \n", + "... ... ... \n", + "39995 19966.0 2013-01-23 09:00:01 \n", + "39996 892.0 2013-01-23 13:13:30 \n", + "39997 2020.0 2013-01-23 09:16:44 \n", + "39998 19914.0 2013-01-23 09:36:07 \n", + "39999 19968.0 2013-02-22 11:23:54 \n", + "\n", + " Title ... AnswerCount \\\n", + "0 Eliciting priors from experts ... 5.0 \n", + "1 What is normality? ... 7.0 \n", + "2 What are some valuable Statistical Analysis op... ... 19.0 \n", + "3 Assessing the significance of differences in d... ... 5.0 \n", + "4 NaN ... NaN \n", + "... ... ... ... \n", + "39995 NaN ... NaN \n", + "39996 NaN ... NaN \n", + "39997 NaN ... NaN \n", + "39998 NaN ... NaN \n", + "39999 are data sets obtained from a Normal distribut... ... 2.0 \n", + "\n", + " CommentCount FavoriteCount LastEditorUserId LastEditDate \\\n", + "0 1 14.0 NaN NaN \n", + "1 1 8.0 88.0 2010-08-07 17:56:44 \n", + "2 4 36.0 183.0 2011-02-12 05:50:03 \n", + "3 2 2.0 NaN NaN \n", + "4 3 NaN 23.0 2010-07-19 19:21:15 \n", + "... ... ... ... ... \n", + "39995 0 NaN NaN NaN \n", + "39996 2 NaN 892.0 2013-01-23 13:13:30 \n", + "39997 0 NaN NaN NaN \n", + "39998 0 NaN NaN NaN \n", + "39999 4 NaN NaN NaN \n", + "\n", + " CommunityOwnedDate ParentId ClosedDate OwnerDisplayName \\\n", + "0 NaN NaN NaN NaN \n", + "1 NaN NaN NaN NaN \n", + "2 2010-07-19 19:13:28 NaN NaN NaN \n", + "3 NaN NaN NaN NaN \n", + "4 2010-07-19 19:14:43 3.0 NaN NaN \n", + "... ... ... ... ... \n", + "39995 NaN 45118.0 NaN NaN \n", + "39996 NaN 48311.0 NaN NaN \n", + "39997 NaN 48247.0 NaN NaN \n", + "39998 NaN 48297.0 NaN NaN \n", + "39999 NaN NaN NaN NaN \n", + "\n", + " LastEditorDisplayName \n", + "0 NaN \n", + "1 NaN \n", + "2 NaN \n", + "3 NaN \n", + "4 NaN \n", + "... ... \n", + "39995 NaN \n", + "39996 NaN \n", + "39997 NaN \n", + "39998 NaN \n", + "39999 NaN \n", + "\n", + "[40000 rows x 21 columns]" + ], + "text/html": [ + "\n", + "

\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IdPostTypeIdAcceptedAnswerIdCreaionDateScoreViewCountBodyOwnerUserIdLasActivityDateTitle...AnswerCountCommentCountFavoriteCountLastEditorUserIdLastEditDateCommunityOwnedDateParentIdClosedDateOwnerDisplayNameLastEditorDisplayName
01115.02010-07-19 19:12:12231278.0<p>How should I elicit prior distributions fro...8.02010-09-15 21:08:26Eliciting priors from experts...5.0114.0NaNNaNNaNNaNNaNNaNNaN
12159.02010-07-19 19:12:57228198.0<p>In many different statistical methods there...24.02012-11-12 09:21:54What is normality?...7.018.088.02010-08-07 17:56:44NaNNaNNaNNaNNaN
2315.02010-07-19 19:13:28543613.0<p>What are some valuable Statistical Analysis...18.02013-05-27 14:48:36What are some valuable Statistical Analysis op......19.0436.0183.02011-02-12 05:50:032010-07-19 19:13:28NaNNaNNaNNaN
341135.02010-07-19 19:13:31135224.0<p>I have two groups of data. Each with a dif...23.02010-09-08 03:00:19Assessing the significance of differences in d......5.022.0NaNNaNNaNNaNNaNNaNNaN
452NaN2010-07-19 19:14:4381NaN<p>The R-project</p>\\n\\n<p><a href=\"http://www...23.02010-07-19 19:21:15NaN...NaN3NaN23.02010-07-19 19:21:152010-07-19 19:14:433.0NaNNaNNaN
..................................................................
39995483212NaN2013-01-23 09:00:010NaN<p>you can use the matlab codes for svm and co...19966.02013-01-23 09:00:01NaN...NaN0NaNNaNNaNNaN45118.0NaNNaNNaN
39996483222NaN2013-01-23 09:09:343NaN<p>I use <a href=\"http://www.gnu.org/software/...892.02013-01-23 13:13:30NaN...NaN2NaN892.02013-01-23 13:13:30NaN48311.0NaNNaNNaN
39997483232NaN2013-01-23 09:16:441NaN<p>If I understand your question correctly, yo...2020.02013-01-23 09:16:44NaN...NaN0NaNNaNNaNNaN48247.0NaNNaNNaN
39998483242NaN2013-01-23 09:36:073NaN<p>Doesn't really help you with your question,...19914.02013-01-23 09:36:07NaN...NaN0NaNNaNNaNNaN48297.0NaNNaNNaN
39999483251NaN2013-01-23 09:44:07-1116.0<p>I have 10 vectors each having 100,000 point...19968.02013-02-22 11:23:54are data sets obtained from a Normal distribut......2.04NaNNaNNaNNaNNaNNaNNaNNaN
\n", + "

40000 rows × 21 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "execution_count": 7 + } + ], + "source": [ + "Posts = pd.read_csv(\"posts_table.csv\")\n", + "Posts" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iwC_Q3-AqtHb" + }, + "source": [ + "#### 5. Rename Id column to postId and OwnerUserId to userId" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 843 + }, + "id": "WRsJqLL2qtHb", + "outputId": "b78496d0-3d6d-4b41-bf03-ff121d3dbda3" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " postId PostTypeId AcceptedAnswerId CreaionDate Score \\\n", + "0 1 1 15.0 2010-07-19 19:12:12 23 \n", + "1 2 1 59.0 2010-07-19 19:12:57 22 \n", + "2 3 1 5.0 2010-07-19 19:13:28 54 \n", + "3 4 1 135.0 2010-07-19 19:13:31 13 \n", + "4 5 2 NaN 2010-07-19 19:14:43 81 \n", + "... ... ... ... ... ... \n", + "39995 48321 2 NaN 2013-01-23 09:00:01 0 \n", + "39996 48322 2 NaN 2013-01-23 09:09:34 3 \n", + "39997 48323 2 NaN 2013-01-23 09:16:44 1 \n", + "39998 48324 2 NaN 2013-01-23 09:36:07 3 \n", + "39999 48325 1 NaN 2013-01-23 09:44:07 -1 \n", + "\n", + " ViewCount Body userId \\\n", + "0 1278.0

How should I elicit prior distributions fro... 8.0 \n", + "1 8198.0

In many different statistical methods there... 24.0 \n", + "2 3613.0

What are some valuable Statistical Analysis... 18.0 \n", + "3 5224.0

I have two groups of data. Each with a dif... 23.0 \n", + "4 NaN

The R-project

\\n\\n

you can use the matlab codes for svm and co... 19966.0 \n", + "39996 NaN

I use If I understand your question correctly, yo... 2020.0 \n", + "39998 NaN

Doesn't really help you with your question,... 19914.0 \n", + "39999 116.0

I have 10 vectors each having 100,000 point... 19968.0 \n", + "\n", + " LasActivityDate Title \\\n", + "0 2010-09-15 21:08:26 Eliciting priors from experts \n", + "1 2012-11-12 09:21:54 What is normality? \n", + "2 2013-05-27 14:48:36 What are some valuable Statistical Analysis op... \n", + "3 2010-09-08 03:00:19 Assessing the significance of differences in d... \n", + "4 2010-07-19 19:21:15 NaN \n", + "... ... ... \n", + "39995 2013-01-23 09:00:01 NaN \n", + "39996 2013-01-23 13:13:30 NaN \n", + "39997 2013-01-23 09:16:44 NaN \n", + "39998 2013-01-23 09:36:07 NaN \n", + "39999 2013-02-22 11:23:54 are data sets obtained from a Normal distribut... \n", + "\n", + " ... AnswerCount CommentCount FavoriteCount LastEditorUserId \\\n", + "0 ... 5.0 1 14.0 NaN \n", + "1 ... 7.0 1 8.0 88.0 \n", + "2 ... 19.0 4 36.0 183.0 \n", + "3 ... 5.0 2 2.0 NaN \n", + "4 ... NaN 3 NaN 23.0 \n", + "... ... ... ... ... ... \n", + "39995 ... NaN 0 NaN NaN \n", + "39996 ... NaN 2 NaN 892.0 \n", + "39997 ... NaN 0 NaN NaN \n", + "39998 ... NaN 0 NaN NaN \n", + "39999 ... 2.0 4 NaN NaN \n", + "\n", + " LastEditDate CommunityOwnedDate ParentId ClosedDate \\\n", + "0 NaN NaN NaN NaN \n", + "1 2010-08-07 17:56:44 NaN NaN NaN \n", + "2 2011-02-12 05:50:03 2010-07-19 19:13:28 NaN NaN \n", + "3 NaN NaN NaN NaN \n", + "4 2010-07-19 19:21:15 2010-07-19 19:14:43 3.0 NaN \n", + "... ... ... ... ... \n", + "39995 NaN NaN 45118.0 NaN \n", + "39996 2013-01-23 13:13:30 NaN 48311.0 NaN \n", + "39997 NaN NaN 48247.0 NaN \n", + "39998 NaN NaN 48297.0 NaN \n", + "39999 NaN NaN NaN NaN \n", + "\n", + " OwnerDisplayName LastEditorDisplayName \n", + "0 NaN NaN \n", + "1 NaN NaN \n", + "2 NaN NaN \n", + "3 NaN NaN \n", + "4 NaN NaN \n", + "... ... ... \n", + "39995 NaN NaN \n", + "39996 NaN NaN \n", + "39997 NaN NaN \n", + "39998 NaN NaN \n", + "39999 NaN NaN \n", + "\n", + "[40000 rows x 21 columns]" + ], + "text/html": [ + "\n", + "

\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
postIdPostTypeIdAcceptedAnswerIdCreaionDateScoreViewCountBodyuserIdLasActivityDateTitle...AnswerCountCommentCountFavoriteCountLastEditorUserIdLastEditDateCommunityOwnedDateParentIdClosedDateOwnerDisplayNameLastEditorDisplayName
01115.02010-07-19 19:12:12231278.0<p>How should I elicit prior distributions fro...8.02010-09-15 21:08:26Eliciting priors from experts...5.0114.0NaNNaNNaNNaNNaNNaNNaN
12159.02010-07-19 19:12:57228198.0<p>In many different statistical methods there...24.02012-11-12 09:21:54What is normality?...7.018.088.02010-08-07 17:56:44NaNNaNNaNNaNNaN
2315.02010-07-19 19:13:28543613.0<p>What are some valuable Statistical Analysis...18.02013-05-27 14:48:36What are some valuable Statistical Analysis op......19.0436.0183.02011-02-12 05:50:032010-07-19 19:13:28NaNNaNNaNNaN
341135.02010-07-19 19:13:31135224.0<p>I have two groups of data. Each with a dif...23.02010-09-08 03:00:19Assessing the significance of differences in d......5.022.0NaNNaNNaNNaNNaNNaNNaN
452NaN2010-07-19 19:14:4381NaN<p>The R-project</p>\\n\\n<p><a href=\"http://www...23.02010-07-19 19:21:15NaN...NaN3NaN23.02010-07-19 19:21:152010-07-19 19:14:433.0NaNNaNNaN
..................................................................
39995483212NaN2013-01-23 09:00:010NaN<p>you can use the matlab codes for svm and co...19966.02013-01-23 09:00:01NaN...NaN0NaNNaNNaNNaN45118.0NaNNaNNaN
39996483222NaN2013-01-23 09:09:343NaN<p>I use <a href=\"http://www.gnu.org/software/...892.02013-01-23 13:13:30NaN...NaN2NaN892.02013-01-23 13:13:30NaN48311.0NaNNaNNaN
39997483232NaN2013-01-23 09:16:441NaN<p>If I understand your question correctly, yo...2020.02013-01-23 09:16:44NaN...NaN0NaNNaNNaNNaN48247.0NaNNaNNaN
39998483242NaN2013-01-23 09:36:073NaN<p>Doesn't really help you with your question,...19914.02013-01-23 09:36:07NaN...NaN0NaNNaNNaNNaN48297.0NaNNaNNaN
39999483251NaN2013-01-23 09:44:07-1116.0<p>I have 10 vectors each having 100,000 point...19968.02013-02-22 11:23:54are data sets obtained from a Normal distribut......2.04NaNNaNNaNNaNNaNNaNNaNNaN
\n", + "

40000 rows × 21 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "execution_count": 8 + } + ], + "source": [ + "Posts = Posts.rename(columns={\"Id\": \"postId\", \"OwnerUserId\": \"userId\"})\n", + "Posts" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QYdpmnz5qtHb" + }, + "source": [ + "#### 6. Define new dataframes for users and posts with the following selected columns:\n", + " **users columns**: userId, Reputation,Views,UpVotes,DownVotes\n", + " **posts columns**: postId, Score,userId,ViewCount,CommentCount" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 831 + }, + "id": "nKnCfL0JqtHc", + "outputId": "11550368-c369-4fd5-e9a5-006db710ff31" + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + " userId Reputation Views UpVotes DownVotes\n", + "0 -1 1 0 5007 1920\n", + "1 2 101 25 3 0\n", + "2 3 101 22 19 0\n", + "3 4 101 11 0 0\n", + "4 5 6792 1145 662 5\n", + "... ... ... ... ... ...\n", + "40320 55743 1 0 0 0\n", + "40321 55744 6 1 0 0\n", + "40322 55745 101 0 0 0\n", + "40323 55746 106 1 0 0\n", + "40324 55747 1 0 0 0\n", + "\n", + "[40325 rows x 5 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdReputationViewsUpVotesDownVotes
0-11050071920
121012530
2310122190
341011100
45679211456625
..................
40320557431000
40321557446100
4032255745101000
4032355746106100
40324557471000
\n", + "

40325 rows × 5 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {} + }, + { + "output_type": "display_data", + "data": { + "text/plain": [ + " postId Score userId ViewCount CommentCount\n", + "0 1 23 8.0 1278.0 1\n", + "1 2 22 24.0 8198.0 1\n", + "2 3 54 18.0 3613.0 4\n", + "3 4 13 23.0 5224.0 2\n", + "4 5 81 23.0 NaN 3\n", + "... ... ... ... ... ...\n", + "39995 48321 0 19966.0 NaN 0\n", + "39996 48322 3 892.0 NaN 2\n", + "39997 48323 1 2020.0 NaN 0\n", + "39998 48324 3 19914.0 NaN 0\n", + "39999 48325 -1 19968.0 116.0 4\n", + "\n", + "[40000 rows x 5 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
postIdScoreuserIdViewCountCommentCount
01238.01278.01
122224.08198.01
235418.03613.04
341323.05224.02
458123.0NaN3
..................
3999548321019966.0NaN0
39996483223892.0NaN2
399974832312020.0NaN0
3999848324319914.0NaN0
3999948325-119968.0116.04
\n", + "

40000 rows × 5 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {} + } + ], + "source": [ + "Users = Users[[\"userId\", \"Reputation\", \"Views\", \"UpVotes\", \"DownVotes\"]]\n", + "Posts = Posts[[\"postId\", \"Score\", \"userId\", \"ViewCount\", \"CommentCount\"]]\n", + "\n", + "display(Users,Posts)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cXr_b9j1qtHc" + }, + "source": [ + "#### 7. Merge both dataframes, users and posts.\n", + "You will need to make a [merge](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.merge.html) of posts and users dataframes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 424 + }, + "id": "eP-D8_vYqtHc", + "outputId": "6074eae2-fc32-4c01-f7f2-69c8c594a004" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " userId Reputation Views UpVotes DownVotes postId Score \\\n", + "0 8.0 6764.0 1089.0 604.0 25.0 1 23 \n", + "1 24.0 344.0 48.0 36.0 1.0 2 22 \n", + "2 18.0 128.0 8.0 16.0 0.0 3 54 \n", + "3 23.0 308.0 52.0 34.0 1.0 4 13 \n", + "4 23.0 308.0 52.0 34.0 1.0 5 81 \n", + "... ... ... ... ... ... ... ... \n", + "39995 19966.0 1.0 1.0 0.0 0.0 48321 0 \n", + "39996 892.0 2772.0 207.0 167.0 2.0 48322 3 \n", + "39997 2020.0 565.0 80.0 81.0 0.0 48323 1 \n", + "39998 19914.0 63.0 6.0 3.0 0.0 48324 3 \n", + "39999 19968.0 1.0 1.0 0.0 0.0 48325 -1 \n", + "\n", + " ViewCount CommentCount \n", + "0 1278.0 1 \n", + "1 8198.0 1 \n", + "2 3613.0 4 \n", + "3 5224.0 2 \n", + "4 NaN 3 \n", + "... ... ... \n", + "39995 NaN 0 \n", + "39996 NaN 2 \n", + "39997 NaN 0 \n", + "39998 NaN 0 \n", + "39999 116.0 4 \n", + "\n", + "[40000 rows x 9 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdReputationViewsUpVotesDownVotespostIdScoreViewCountCommentCount
08.06764.01089.0604.025.01231278.01
124.0344.048.036.01.02228198.01
218.0128.08.016.00.03543613.04
323.0308.052.034.01.04135224.02
423.0308.052.034.01.0581NaN3
..............................
3999519966.01.01.00.00.0483210NaN0
39996892.02772.0207.0167.02.0483223NaN2
399972020.0565.080.081.00.0483231NaN0
3999819914.063.06.03.00.0483243NaN0
3999919968.01.01.00.00.048325-1116.04
\n", + "

40000 rows × 9 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "execution_count": 13 + } + ], + "source": [ + "Merged = Users.merge(Posts, how=\"right\")\n", + "\n", + "Merged" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "NJCiQllsqtHc" + }, + "source": [ + "#### 8. How many missing values do you have in your merged dataframe? On which columns?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ws26wSBFqtHc", + "outputId": "bf94bee4-a0d6-4414-dc65-ac0c2ec29539" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "userId 1038\n", + "Reputation 1038\n", + "Views 1038\n", + "UpVotes 1038\n", + "DownVotes 1038\n", + "postId 0\n", + "Score 0\n", + "ViewCount 24105\n", + "CommentCount 0\n", + "dtype: int64\n" + ] + } + ], + "source": [ + "missing_values = Merged.isnull().sum()\n", + "print(missing_values)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tLFjJEqzqtHc" + }, + "source": [ + "#### 9. You will need to make something with missing values. Will you clean or filling them? Explain.\n", + "**Remember** to check the results of your code before passing to the next step" + ] + }, + { + "cell_type": "markdown", + "source": [ + "I will clean the value and delete the rows without user data." + ], + "metadata": { + "id": "9Ff5NK_Ixq1E" + } + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zfKk_b89qtHc", + "outputId": "b3a90341-0b4d-4396-f10e-d6d0ace4fbe9" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0 False\n", + "1 False\n", + "2 False\n", + "3 False\n", + "4 False\n", + " ... \n", + "39995 False\n", + "39996 False\n", + "39997 False\n", + "39998 False\n", + "39999 False\n", + "Name: userId, Length: 40000, dtype: bool" + ] + }, + "metadata": {}, + "execution_count": 20 + } + ], + "source": [ + "users_null = Merged[\"userId\"].isnull()\n", + "\n", + "users_null" + ] + }, + { + "cell_type": "code", + "source": [ + "import pandas as pd\n", + "\n", + "Merged = Merged.dropna(subset=['userId'])\n", + "\n", + "Merged = Merged.reset_index(drop=True)\n", + "\n", + "Merged\n" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 424 + }, + "id": "mMgJNzpG7CZC", + "outputId": "8b3c9f25-7fa1-4d8a-d2be-6c1135c8ac2d" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " userId Reputation Views UpVotes DownVotes postId Score \\\n", + "0 8.0 6764.0 1089.0 604.0 25.0 1 23 \n", + "1 24.0 344.0 48.0 36.0 1.0 2 22 \n", + "2 18.0 128.0 8.0 16.0 0.0 3 54 \n", + "3 23.0 308.0 52.0 34.0 1.0 4 13 \n", + "4 23.0 308.0 52.0 34.0 1.0 5 81 \n", + "... ... ... ... ... ... ... ... \n", + "38957 19966.0 1.0 1.0 0.0 0.0 48321 0 \n", + "38958 892.0 2772.0 207.0 167.0 2.0 48322 3 \n", + "38959 2020.0 565.0 80.0 81.0 0.0 48323 1 \n", + "38960 19914.0 63.0 6.0 3.0 0.0 48324 3 \n", + "38961 19968.0 1.0 1.0 0.0 0.0 48325 -1 \n", + "\n", + " ViewCount CommentCount \n", + "0 1278.0 1 \n", + "1 8198.0 1 \n", + "2 3613.0 4 \n", + "3 5224.0 2 \n", + "4 NaN 3 \n", + "... ... ... \n", + "38957 NaN 0 \n", + "38958 NaN 2 \n", + "38959 NaN 0 \n", + "38960 NaN 0 \n", + "38961 116.0 4 \n", + "\n", + "[38962 rows x 9 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userIdReputationViewsUpVotesDownVotespostIdScoreViewCountCommentCount
08.06764.01089.0604.025.01231278.01
124.0344.048.036.01.02228198.01
218.0128.08.016.00.03543613.04
323.0308.052.034.01.04135224.02
423.0308.052.034.01.0581NaN3
..............................
3895719966.01.01.00.00.0483210NaN0
38958892.02772.0207.0167.02.0483223NaN2
389592020.0565.080.081.00.0483231NaN0
3896019914.063.06.03.00.0483243NaN0
3896119968.01.01.00.00.048325-1116.04
\n", + "

38962 rows × 9 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "execution_count": 32 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "u69VJ7SPqtHc" + }, + "source": [ + "#### 10. Adjust the data types in order to avoid future issues. Which ones should be changed?" + ] + }, + { + "cell_type": "code", + "source": [ + "Merged.dtypes" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "LusSR1mF7aU3", + "outputId": "c5dc4eb4-4371-43eb-94b5-bb25338e1d03" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "userId float64\n", + "Reputation float64\n", + "Views float64\n", + "UpVotes float64\n", + "DownVotes float64\n", + "postId int64\n", + "Score int64\n", + "ViewCount float64\n", + "CommentCount int64\n", + "dtype: object" + ] + }, + "metadata": {}, + "execution_count": 34 + } + ] + }, + { + "cell_type": "code", + "source": [ + "Merged[[\"userId\", \"Reputation\", \"Views\", \"UpVotes\", \"DownVotes\", \"ViewCount\"]] = Merged[[\"userId\", \"Reputation\", \"Views\", \"UpVotes\", \"DownVotes\", \"ViewCount\"]].fillna(0).astype('int64')\n", + "Merged.dtypes" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "WIcKKVCm7wrz", + "outputId": "5bfeb03e-1dfc-495e-dff8-ec331a5cc5b8" + }, + "execution_count": null, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "userId int64\n", + "Reputation int64\n", + "Views int64\n", + "UpVotes int64\n", + "DownVotes int64\n", + "postId int64\n", + "Score int64\n", + "ViewCount int64\n", + "CommentCount int64\n", + "dtype: object" + ] + }, + "metadata": {}, + "execution_count": 40 + } + ] + }, + { + "cell_type": "code", + "source": [], + "metadata": { + "id": "A6Q0-RKK75SQ" + }, + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.8" + }, + "colab": { + "provenance": [], + "include_colab_link": true + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file