diff --git a/.travis.yml b/.travis.yml
index 1db80743..38f60163 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -14,6 +14,6 @@ services:
- postgresql
env:
global:
- - PGPORT=5432
+ - SB_TEST_PGPORT=5432
script:
- make test-travis
diff --git a/docs/.gitattributes b/docs/.gitattributes
index d9d68857..19c35d26 100644
--- a/docs/.gitattributes
+++ b/docs/.gitattributes
@@ -1,3 +1,2 @@
-*.ipynb filter=nbstripout
*.ipynb diff=ipynb
diff --git a/docs/index.rst b/docs/index.rst
index 7f27327c..6ef37a36 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -3,7 +3,7 @@
:hidden:
intro.Rmd
- intro_sql.Rmd
+ intro_sql.ipynb
.. toctree::
:caption: Core One-table Verbs
diff --git a/docs/intro_sql.Rmd b/docs/intro_sql.Rmd
index 8f4d3909..77c56131 100644
--- a/docs/intro_sql.Rmd
+++ b/docs/intro_sql.Rmd
@@ -12,38 +12,131 @@ jupyter:
name: python3
---
+```{python nbsphinx=hidden}
+import matplotlib.cbook
+
+import warnings
+import plotnine
+warnings.filterwarnings(module='plotnine*', action='ignore')
+warnings.filterwarnings(module='matplotlib*', action='ignore')
+
+# %matplotlib inline
+```
+
# Using to query SQL
+
+# Setting up
+
```{python}
-from sqlalchemy import create_engine
-from siuba.data import mtcars
import pandas as pd
+from siuba.tests.helpers import copy_to_sql
+from siuba import *
+from siuba.dply.vector import lag, desc, row_number
+from siuba.dply.string import str_c
-engine = create_engine('sqlite:///:memory:', echo=False)
+tv_ratings = pd.read_csv(
+ "https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2019/2019-01-08/IMDb_Economist_tv_ratings.csv",
+ parse_dates = ["date"]
+ )
-# note that mtcars is a pandas DataFrame
-mtcars.to_sql('mtcars', engine)
```
```{python}
-from siuba import *
-from siuba.sql import LazyTbl, show_query, collect
+tbl_ratings = copy_to_sql(tv_ratings, "tv_ratings", "postgresql://postgres:@localhost:5433/postgres")
-tbl_mtcars = LazyTbl(engine, 'mtcars')
```
```{python}
-tbl_mtcars
+tbl_ratings
+
```
+## Inspecting a single show
+
```{python}
-tbl_mtcars >> filter(_.hp > 250) >> collect()
+buffy = (tbl_ratings
+ >> filter(_.title == "Buffy the Vampire Slayer")
+ >> collect()
+ )
+
+buffy
+```
+
+```{python}
+buffy >> summarize(avg_rating = _.av_rating.mean())
```
+## Average rating per show, along with dates
+
```{python}
-(tbl_mtcars
- >> group_by(_.cyl)
- >> summarize(avg_mpg = _.mpg.mean())
+avg_ratings = (tbl_ratings
+ >> group_by(_.title)
+ >> summarize(
+ avg_rating = _.av_rating.mean(),
+ date_range = str_c(_.date.dt.year.max(), " - ", _.date.dt.year.min())
+ )
+ )
+
+avg_ratings
+```
+
+## Biggest changes in ratings between two seasons
+
+```{python}
+top_4_shifts = (tbl_ratings
+ >> group_by(_.title)
+ >> mutate(rating_shift = _.av_rating - lag(_.av_rating))
+ >> summarize(
+ max_shift = _.rating_shift.max()
+ )
+ >> arrange(-_.max_shift)
+ >> head(4)
+ )
+
+top_4_shifts
+```
+
+```{python}
+big_shift_series = (top_4_shifts
+ >> select(_.title)
+ >> inner_join(_, tbl_ratings, "title")
>> collect()
)
+
+from plotnine import *
+
+(big_shift_series
+ >> ggplot(aes("seasonNumber", "av_rating"))
+ + geom_point()
+ + geom_line()
+ + facet_wrap("~ title")
+ + labs(
+ title = "Seasons with Biggest Shifts in Ratings",
+ y = "Average rating",
+ x = "Season"
+ )
+ )
+```
+
+## Do we have full data for each season?
+
+```{python}
+mismatches = (tbl_ratings
+ >> arrange(_.title, _.seasonNumber)
+ >> group_by(_.title)
+ >> mutate(
+ row = row_number(_),
+ mismatch = _.row != _.seasonNumber
+ )
+ >> filter(_.mismatch.any())
+ >> ungroup()
+ )
+
+
+mismatches
+```
+
+```{python}
+mismatches >> distinct(_.title) >> count() >> collect()
```
diff --git a/docs/intro_sql.ipynb b/docs/intro_sql.ipynb
new file mode 100644
index 00000000..fb4712ea
--- /dev/null
+++ b/docs/intro_sql.ipynb
@@ -0,0 +1,877 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "nbsphinx": "hidden"
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib.cbook\n",
+ "\n",
+ "import warnings\n",
+ "import plotnine\n",
+ "warnings.filterwarnings(module='plotnine*', action='ignore')\n",
+ "warnings.filterwarnings(module='matplotlib*', action='ignore')\n",
+ "\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Using to query SQL"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Setting up"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "from siuba.tests.helpers import copy_to_sql\n",
+ "from siuba import *\n",
+ "from siuba.dply.vector import lag, desc, row_number\n",
+ "from siuba.dply.string import str_c\n",
+ "\n",
+ "tv_ratings = pd.read_csv(\n",
+ " \"https://raw.githubusercontent.com/rfordatascience/tidytuesday/master/data/2019/2019-01-08/IMDb_Economist_tv_ratings.csv\",\n",
+ " parse_dates = [\"date\"]\n",
+ " )\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tbl_ratings = copy_to_sql(tv_ratings, \"tv_ratings\", \"postgresql://postgres:@localhost:5433/postgres\")\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "# Source: lazy query\n",
+ "# DB Conn: Engine(postgresql://postgres:***@localhost:5433/postgres)\n",
+ "# Preview:\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " titleId | \n",
+ " seasonNumber | \n",
+ " title | \n",
+ " date | \n",
+ " av_rating | \n",
+ " share | \n",
+ " genres | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " tt2879552 | \n",
+ " 1 | \n",
+ " 11.22.63 | \n",
+ " 2016-03-10 | \n",
+ " 8.4890 | \n",
+ " 0.51 | \n",
+ " Drama,Mystery,Sci-Fi | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " tt3148266 | \n",
+ " 1 | \n",
+ " 12 Monkeys | \n",
+ " 2015-02-27 | \n",
+ " 8.3407 | \n",
+ " 0.46 | \n",
+ " Adventure,Drama,Mystery | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " tt3148266 | \n",
+ " 2 | \n",
+ " 12 Monkeys | \n",
+ " 2016-05-30 | \n",
+ " 8.8196 | \n",
+ " 0.25 | \n",
+ " Adventure,Drama,Mystery | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " tt3148266 | \n",
+ " 3 | \n",
+ " 12 Monkeys | \n",
+ " 2017-05-19 | \n",
+ " 9.0369 | \n",
+ " 0.19 | \n",
+ " Adventure,Drama,Mystery | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " tt3148266 | \n",
+ " 4 | \n",
+ " 12 Monkeys | \n",
+ " 2018-06-26 | \n",
+ " 9.1363 | \n",
+ " 0.38 | \n",
+ " Adventure,Drama,Mystery | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
# .. may have more rows
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " titleId | \n",
+ " seasonNumber | \n",
+ " title | \n",
+ " date | \n",
+ " av_rating | \n",
+ " share | \n",
+ " genres | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " tt0118276 | \n",
+ " 1 | \n",
+ " Buffy the Vampire Slayer | \n",
+ " 1997-04-14 | \n",
+ " 7.9629 | \n",
+ " 11.70 | \n",
+ " Action,Drama,Fantasy | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " tt0118276 | \n",
+ " 2 | \n",
+ " Buffy the Vampire Slayer | \n",
+ " 1997-12-31 | \n",
+ " 8.4191 | \n",
+ " 19.41 | \n",
+ " Action,Drama,Fantasy | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " tt0118276 | \n",
+ " 3 | \n",
+ " Buffy the Vampire Slayer | \n",
+ " 1999-01-29 | \n",
+ " 8.6233 | \n",
+ " 17.12 | \n",
+ " Action,Drama,Fantasy | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " tt0118276 | \n",
+ " 4 | \n",
+ " Buffy the Vampire Slayer | \n",
+ " 2000-01-19 | \n",
+ " 8.2205 | \n",
+ " 16.19 | \n",
+ " Action,Drama,Fantasy | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " tt0118276 | \n",
+ " 5 | \n",
+ " Buffy the Vampire Slayer | \n",
+ " 2001-01-12 | \n",
+ " 8.3028 | \n",
+ " 11.99 | \n",
+ " Action,Drama,Fantasy | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " tt0118276 | \n",
+ " 6 | \n",
+ " Buffy the Vampire Slayer | \n",
+ " 2002-01-29 | \n",
+ " 8.1008 | \n",
+ " 8.45 | \n",
+ " Action,Drama,Fantasy | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " tt0118276 | \n",
+ " 7 | \n",
+ " Buffy the Vampire Slayer | \n",
+ " 2003-01-18 | \n",
+ " 8.0460 | \n",
+ " 9.89 | \n",
+ " Action,Drama,Fantasy | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " avg_rating | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 8.239343 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
# Source: lazy query\n",
+ "# DB Conn: Engine(postgresql://postgres:***@localhost:5433/postgres)\n",
+ "# Preview:\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " title | \n",
+ " avg_rating | \n",
+ " date_range | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " Friends from College | \n",
+ " 6.875100 | \n",
+ " 2017 - 2017 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " Better Things | \n",
+ " 8.133150 | \n",
+ " 2017 - 2016 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " How to Get Away with Murder | \n",
+ " 8.762340 | \n",
+ " 2018 - 2014 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " Dexter | \n",
+ " 8.582400 | \n",
+ " 2013 - 2006 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " Queen of the South | \n",
+ " 8.574733 | \n",
+ " 2018 - 2016 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
# .. may have more rows
# Source: lazy query\n",
+ "# DB Conn: Engine(postgresql://postgres:***@localhost:5433/postgres)\n",
+ "# Preview:\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " title | \n",
+ " max_shift | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " Third Watch | \n",
+ " 4.8500 | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " Are You Afraid of the Dark? | \n",
+ " 2.3430 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " Lethal Weapon | \n",
+ " 2.3070 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " Law & Order: Special Victims Unit | \n",
+ " 2.0508 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
# .. may have more rows
"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/plain": [
+ ""
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "big_shift_series = (top_4_shifts\n",
+ " >> select(_.title)\n",
+ " >> inner_join(_, tbl_ratings, \"title\")\n",
+ " >> collect()\n",
+ " )\n",
+ "\n",
+ "from plotnine import *\n",
+ "\n",
+ "(big_shift_series\n",
+ " >> ggplot(aes(\"seasonNumber\", \"av_rating\"))\n",
+ " + geom_point()\n",
+ " + geom_line()\n",
+ " + facet_wrap(\"~ title\")\n",
+ " + labs(\n",
+ " title = \"Seasons with Biggest Shifts in Ratings\",\n",
+ " y = \"Average rating\",\n",
+ " x = \"Season\"\n",
+ " )\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Do we have full data for each season?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "# Source: lazy query\n",
+ "# DB Conn: Engine(postgresql://postgres:***@localhost:5433/postgres)\n",
+ "# Preview:\n",
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " title | \n",
+ " titleId | \n",
+ " seasonNumber | \n",
+ " date | \n",
+ " av_rating | \n",
+ " share | \n",
+ " genres | \n",
+ " row | \n",
+ " mismatch | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 7th Heaven | \n",
+ " tt0115083 | \n",
+ " 1 | \n",
+ " 1996-08-26 | \n",
+ " 7.700 | \n",
+ " 0.10 | \n",
+ " Drama,Family,Romance | \n",
+ " 1 | \n",
+ " False | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " 7th Heaven | \n",
+ " tt0115083 | \n",
+ " 10 | \n",
+ " 2006-05-08 | \n",
+ " 6.300 | \n",
+ " 0.01 | \n",
+ " Drama,Family,Romance | \n",
+ " 2 | \n",
+ " True | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " ABC Afterschool Specials | \n",
+ " tt0202179 | \n",
+ " 25 | \n",
+ " 1996-09-12 | \n",
+ " 3.300 | \n",
+ " 0.10 | \n",
+ " Adventure,Comedy,Drama | \n",
+ " 1 | \n",
+ " True | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " American Gothic | \n",
+ " tt5257744 | \n",
+ " 1 | \n",
+ " 2016-08-05 | \n",
+ " 7.535 | \n",
+ " 0.07 | \n",
+ " Crime,Drama,Mystery | \n",
+ " 1 | \n",
+ " False | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " American Gothic | \n",
+ " tt0111880 | \n",
+ " 1 | \n",
+ " 1995-09-22 | \n",
+ " 7.800 | \n",
+ " 0.08 | \n",
+ " Drama,Horror,Thriller | \n",
+ " 2 | \n",
+ " True | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
# .. may have more rows
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " n | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 54 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
" | \n",
- " id | \n",
" user_id | \n",
+ " id | \n",
" email_address | \n",
" num | \n",
- " anon_2 | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
- " 2 | \n",
" 1 | \n",
+ " 2 | \n",
" jack@msn.com | \n",
" 2 | \n",
- " 1 | \n",
"
\n",
" \n",
"\n",
""
],
"text/plain": [
- " id user_id email_address num anon_2\n",
- "0 2 1 jack@msn.com 2 1"
+ " user_id id email_address num\n",
+ "0 1 2 jack@msn.com 2"
]
},
"execution_count": 2,
@@ -180,6 +178,26 @@
"#tbl_addresses >> group_by(_, \"user_id\") >> mutate(_, num = dense_rank(_.id)) >> show_query(_)"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ ">"
+ ]
+ },
+ "execution_count": 3,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "sql.functions.sum().over"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -189,7 +207,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 4,
"metadata": {},
"outputs": [
{
@@ -210,7 +228,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 5,
"metadata": {},
"outputs": [
{
@@ -232,7 +250,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 6,
"metadata": {},
"outputs": [
{
@@ -264,7 +282,7 @@
},
{
"cell_type": "code",
- "execution_count": 6,
+ "execution_count": 7,
"metadata": {},
"outputs": [
{
@@ -288,14 +306,14 @@
},
{
"cell_type": "code",
- "execution_count": 7,
+ "execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "SELECT anon_1.id, anon_1.user_id, anon_1.email_address \n",
+ "SELECT anon_1.user_id, anon_1.id, anon_1.email_address \n",
"FROM (SELECT anon_2.id AS id, anon_2.user_id AS user_id, anon_2.email_address AS email_address \n",
"FROM (SELECT addresses.id AS id, addresses.user_id AS user_id, addresses.email_address AS email_address \n",
"FROM addresses) AS anon_2) AS anon_1 \n",
@@ -313,18 +331,18 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "SELECT anon_1.id, anon_1.user_id, anon_1.email_address, anon_1.anon_2 \n",
- "FROM (SELECT anon_3.id AS id, anon_3.user_id AS user_id, anon_3.email_address AS email_address, dense_rank() OVER (PARTITION BY anon_3.user_id ORDER BY anon_3.id) AS anon_2 \n",
+ "SELECT anon_1.user_id, anon_1.id, anon_1.email_address \n",
+ "FROM (SELECT anon_2.id AS id, anon_2.user_id AS user_id, anon_2.email_address AS email_address, dense_rank() OVER (PARTITION BY anon_2.user_id ORDER BY anon_2.id) AS anon_3 \n",
"FROM (SELECT addresses.id AS id, addresses.user_id AS user_id, addresses.email_address AS email_address \n",
- "FROM addresses) AS anon_3) AS anon_1 \n",
- "WHERE anon_1.anon_2 > 1\n"
+ "FROM addresses) AS anon_2) AS anon_1 \n",
+ "WHERE anon_1.anon_3 > 1\n"
]
},
{
@@ -348,38 +366,35 @@
" \n",
" \n",
" | \n",
- " id | \n",
" user_id | \n",
+ " id | \n",
" email_address | \n",
- " anon_2 | \n",
"
\n",
" \n",
" \n",
" \n",
" | 0 | \n",
- " 2 | \n",
" 1 | \n",
- " jack@msn.com | \n",
" 2 | \n",
+ " jack@msn.com | \n",
"
\n",
" \n",
" | 1 | \n",
- " 4 | \n",
" 2 | \n",
+ " 4 | \n",
" wendy@aol.com | \n",
- " 2 | \n",
"
\n",
" \n",
"\n",
""
],
"text/plain": [
- " id user_id email_address anon_2\n",
- "0 2 1 jack@msn.com 2\n",
- "1 4 2 wendy@aol.com 2"
+ " user_id id email_address\n",
+ "0 1 2 jack@msn.com\n",
+ "1 2 4 wendy@aol.com"
]
},
- "execution_count": 8,
+ "execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
@@ -404,7 +419,7 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 10,
"metadata": {},
"outputs": [
{
@@ -461,7 +476,7 @@
"1 1 1.5"
]
},
- "execution_count": 9,
+ "execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
@@ -479,15 +494,16 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "SELECT avg(addresses.id + 1) AS m_id \n",
- "FROM addresses\n"
+ "SELECT avg(anon_1.id2) AS m_id \n",
+ "FROM (SELECT addresses.id AS id, addresses.user_id AS user_id, addresses.email_address AS email_address, addresses.id + 1 AS id2 \n",
+ "FROM addresses) AS anon_1\n"
]
}
],
@@ -504,7 +520,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 12,
"metadata": {},
"outputs": [
{
@@ -566,7 +582,7 @@
"1 2 0 2"
]
},
- "execution_count": 11,
+ "execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
@@ -591,16 +607,16 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 13,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "SELECT anon_1.id AS id_x, anon_1.user_id, anon_1.email_address, anon_2.id AS id_y, anon_2.name, anon_2.fullname \n",
+ "SELECT anon_1.id, anon_1.user_id, anon_1.email_address, anon_2.fullname, anon_2.name \n",
"FROM (SELECT addresses.id AS id, addresses.user_id AS user_id, addresses.email_address AS email_address \n",
- "FROM addresses) AS anon_1 JOIN (SELECT users.id AS id, users.name AS name, users.fullname AS fullname \n",
+ "FROM addresses) AS anon_1 LEFT OUTER JOIN (SELECT users.id AS id, users.name AS name, users.fullname AS fullname \n",
"FROM users) AS anon_2 ON anon_1.user_id = anon_2.id\n"
]
},
@@ -625,12 +641,11 @@
" \n",
" \n",
" | \n",
- " id_x | \n",
+ " id | \n",
" user_id | \n",
" email_address | \n",
- " id_y | \n",
- " name | \n",
" fullname | \n",
+ " name | \n",
"
\n",
" \n",
" \n",
@@ -639,50 +654,46 @@
" 1 | \n",
" 1 | \n",
" jack@yahoo.com | \n",
- " 1 | \n",
- " jack | \n",
" Jack Jones | \n",
+ " jack | \n",
" \n",
" \n",
" | 1 | \n",
" 2 | \n",
" 1 | \n",
" jack@msn.com | \n",
- " 1 | \n",
- " jack | \n",
" Jack Jones | \n",
+ " jack | \n",
"
\n",
" \n",
" | 2 | \n",
" 3 | \n",
" 2 | \n",
" www@www.org | \n",
- " 2 | \n",
- " wendy | \n",
" Wendy Williams | \n",
+ " wendy | \n",
"
\n",
" \n",
" | 3 | \n",
" 4 | \n",
" 2 | \n",
" wendy@aol.com | \n",
- " 2 | \n",
- " wendy | \n",
" Wendy Williams | \n",
+ " wendy | \n",
"
\n",
" \n",
"\n",
""
],
"text/plain": [
- " id_x user_id email_address id_y name fullname\n",
- "0 1 1 jack@yahoo.com 1 jack Jack Jones\n",
- "1 2 1 jack@msn.com 1 jack Jack Jones\n",
- "2 3 2 www@www.org 2 wendy Wendy Williams\n",
- "3 4 2 wendy@aol.com 2 wendy Wendy Williams"
+ " id user_id email_address fullname name\n",
+ "0 1 1 jack@yahoo.com Jack Jones jack\n",
+ "1 2 1 jack@msn.com Jack Jones jack\n",
+ "2 3 2 www@www.org Wendy Williams wendy\n",
+ "3 4 2 wendy@aol.com Wendy Williams wendy"
]
},
- "execution_count": 12,
+ "execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
@@ -708,7 +719,7 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 14,
"metadata": {},
"outputs": [
{
@@ -787,7 +798,7 @@
"3 4 2 wendy@aol.com 1"
]
},
- "execution_count": 13,
+ "execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
@@ -811,7 +822,7 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 15,
"metadata": {},
"outputs": [
{
@@ -867,7 +878,7 @@
"0 1 1 jack@yahoo.com"
]
},
- "execution_count": 14,
+ "execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
@@ -892,7 +903,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
@@ -971,7 +982,7 @@
"3 4 2 wendy@aol.com 0"
]
},
- "execution_count": 15,
+ "execution_count": 16,
"metadata": {},
"output_type": "execute_result"
}
@@ -997,14 +1008,14 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"█─'__call__'\n",
- "├─\n",
+ "├─\n",
"├─_\n",
"└─█─''\n",
" └─█─'__call__'\n",
@@ -1012,7 +1023,7 @@
" └─{_.id > 1: 'yeah', True: 'no'}"
]
},
- "execution_count": 16,
+ "execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
@@ -1031,7 +1042,7 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": 18,
"metadata": {},
"outputs": [
{
@@ -1059,7 +1070,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 19,
"metadata": {},
"outputs": [
{
@@ -1127,7 +1138,7 @@
"2 3 2 www@www.org"
]
},
- "execution_count": 18,
+ "execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
@@ -1149,7 +1160,7 @@
},
{
"cell_type": "code",
- "execution_count": 19,
+ "execution_count": 20,
"metadata": {},
"outputs": [
{
@@ -1223,7 +1234,7 @@
"3 4 2 wendy@aol.com"
]
},
- "execution_count": 19,
+ "execution_count": 20,
"metadata": {},
"output_type": "execute_result"
}
@@ -1245,7 +1256,7 @@
},
{
"cell_type": "code",
- "execution_count": 20,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
@@ -1302,7 +1313,7 @@
"1 1 2"
]
},
- "execution_count": 20,
+ "execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
@@ -1317,7 +1328,7 @@
},
{
"cell_type": "code",
- "execution_count": 21,
+ "execution_count": 22,
"metadata": {},
"outputs": [
{
@@ -1386,7 +1397,7 @@
"3 wendy@aol.com 1"
]
},
- "execution_count": 21,
+ "execution_count": 22,
"metadata": {},
"output_type": "execute_result"
}
@@ -1416,7 +1427,7 @@
},
{
"cell_type": "code",
- "execution_count": 22,
+ "execution_count": 23,
"metadata": {},
"outputs": [
{
@@ -1465,7 +1476,7 @@
"1 1 2"
]
},
- "execution_count": 22,
+ "execution_count": 23,
"metadata": {},
"output_type": "execute_result"
}
@@ -1495,7 +1506,7 @@
},
{
"cell_type": "code",
- "execution_count": 23,
+ "execution_count": 24,
"metadata": {},
"outputs": [
{
@@ -1518,6 +1529,55 @@
"## SQL escapes"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Window functions"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 25,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "SELECT addresses.id, addresses.user_id, addresses.email_address, sum(addresses.user_id) OVER (ORDER BY addresses.id DESC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS cumsum \n",
+ "FROM addresses ORDER BY addresses.id DESC, cumsum\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "# Source: lazy query\n",
+ "# DB Conn: Engine(postgresql://postgres:***@localhost:5433/postgres)\n",
+ "# Preview:\n",
+ " id user_id email_address cumsum\n",
+ "0 4 2 wendy@aol.com 2\n",
+ "1 3 2 www@www.org 4\n",
+ "2 2 1 jack@msn.com 5\n",
+ "3 1 1 jack@yahoo.com 6\n",
+ "# .. may have more rows"
+ ]
+ },
+ "execution_count": 25,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from siuba.dply.vector import desc\n",
+ "(tbl_addresses\n",
+ " >> arrange(_.id.desc())\n",
+ " >> mutate(cumsum = _.user_id.cumsum())\n",
+ " >> arrange(_.cumsum)\n",
+ " >> show_query()\n",
+ " )"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -1534,7 +1594,7 @@
},
{
"cell_type": "code",
- "execution_count": 24,
+ "execution_count": 26,
"metadata": {},
"outputs": [
{
@@ -1613,7 +1673,7 @@
"3 4 2 wendy@aol.com 4.0"
]
},
- "execution_count": 24,
+ "execution_count": 26,
"metadata": {},
"output_type": "execute_result"
}
@@ -1635,7 +1695,7 @@
},
{
"cell_type": "code",
- "execution_count": 25,
+ "execution_count": 27,
"metadata": {},
"outputs": [
{
@@ -1698,7 +1758,7 @@
"1 2 wendy Wendy Williams 3"
]
},
- "execution_count": 25,
+ "execution_count": 27,
"metadata": {},
"output_type": "execute_result"
}
@@ -1727,7 +1787,7 @@
},
{
"cell_type": "code",
- "execution_count": 26,
+ "execution_count": 28,
"metadata": {},
"outputs": [
{
@@ -1790,7 +1850,7 @@
"1 2 wendy Wendy Williams 3"
]
},
- "execution_count": 26,
+ "execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
@@ -1818,7 +1878,7 @@
},
{
"cell_type": "code",
- "execution_count": 27,
+ "execution_count": 29,
"metadata": {},
"outputs": [
{
@@ -1833,7 +1893,7 @@
"# .. may have more rows"
]
},
- "execution_count": 27,
+ "execution_count": 29,
"metadata": {},
"output_type": "execute_result"
}
diff --git a/requirements.txt b/requirements.txt
index cb3bd17b..1507e4ec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,10 +2,12 @@ numpy==1.16.1
pandas==0.24.1
python-dateutil==2.8.0
pytz==2018.9
+scipy==1.2.1
six==1.12.0
SQLAlchemy==1.2.17
nbval==0.9.1
# tests
+pytest==4.4.2
psycopg2==2.8.2
# only used for iris dataset
scikit-learn==0.20.2
@@ -13,3 +15,5 @@ scikit-learn==0.20.2
nbsphinx==0.4.2
jupytext==1.1.1
gapminder==0.1
+matplotlib==3.1.0
+plotnine==0.5.1
diff --git a/siuba/dply/string.py b/siuba/dply/string.py
new file mode 100644
index 00000000..b4099868
--- /dev/null
+++ b/siuba/dply/string.py
@@ -0,0 +1,33 @@
+import pandas as pd
+import numpy as np
+from functools import singledispatch
+import itertools
+
+from ..siu import Symbolic, create_sym_call,Call
+
+
+def register_symbolic(f):
+ # TODO: don't use singledispatch if it has already been done
+ f = singledispatch(f)
+ @f.register(Symbolic)
+ def _dispatch_symbol(__data, *args, **kwargs):
+ return create_sym_call(f, __data.source, *args, **kwargs)
+
+ return f
+
+def _coerce_to_str(x):
+ if isinstance(x, (pd.Series, np.ndarray)):
+ return x.astype(str)
+ elif not np.ndim(x) < 2:
+ raise ValueError("np.ndim must be less than 2, but is %s" %np.ndim(x))
+
+ return pd.Series(x, dtype = str)
+
+
+@register_symbolic
+def str_c(x, *args, sep = "", collapse = None):
+ all_args = itertools.chain([x], args)
+ strings = list(map(_coerce_to_str, all_args))
+
+ return np.sum(strings, axis = 0)
+
diff --git a/siuba/dply/vector.py b/siuba/dply/vector.py
index 54a07319..dd000e4c 100644
--- a/siuba/dply/vector.py
+++ b/siuba/dply/vector.py
@@ -32,7 +32,7 @@ def cummean(x):
@register_symbolic
def desc(x):
- NotImplementedError("Use minus sign in arrange instead (e.g. -_.somecol)")
+ return x.sort_values()
@register_symbolic
@@ -61,7 +61,7 @@ def row_number(x):
n = x.shape[0]
else:
n = len(x)
- return np.arange(n)
+ return np.arange(1, n + 1)
@register_symbolic
@@ -91,7 +91,7 @@ def lead(x, n = 1, default = None):
@register_symbolic
-def lag():
+def lag(x, n = 1, default = None):
res = x.shift(n)
if default is not None:
diff --git a/siuba/dply/verbs.py b/siuba/dply/verbs.py
index 60e6787a..b5b2fde4 100644
--- a/siuba/dply/verbs.py
+++ b/siuba/dply/verbs.py
@@ -19,9 +19,10 @@
"nest", "unnest",
"expand", "complete",
# Joins ----
- "join", "inner_join", "left_join", "right_join", "semi_join", "full_join",
+ "join", "inner_join", "full_join", "left_join", "right_join", "semi_join", "anti_join",
# TODO: move to vectors
"if_else", "case_when",
+ "collect", "show_query"
)
__all__ = [*DPLY_FUNCTIONS, "Pipeable", "pipe"]
@@ -206,6 +207,20 @@ def raise_type_error(f):
types = ", ".join(map(str, f.registry.keys()))
))
+# Collect and show_query =========
+
+@pipe_no_args
+@singledispatch2((DataFrame, DataFrameGroupBy))
+def collect(__data, *args, **kwargs):
+ # simply return DataFrame, since requires no execution
+ return __data
+
+
+@pipe_no_args
+@singledispatch2((DataFrame, DataFrameGroupBy))
+def show_query(__data, simplify = False):
+ print("No query to show for a DataFrame")
+ return __data
# Mutate ======================================================================
@@ -538,6 +553,11 @@ def var_create(*args):
@singledispatch2(DataFrame)
def select(__data, *args, **kwargs):
+ if kwargs:
+ raise NotImplementedError(
+ "Using kwargs in select not currently supported. "
+ "Use _.newname == _.oldname instead"
+ )
var_list = var_create(*args)
od = var_select(__data.columns, *var_list)
@@ -557,13 +577,15 @@ def _select(__data, *args, **kwargs):
@singledispatch2(DataFrame)
def rename(__data, **kwargs):
# TODO: allow names with spaces, etc..
- col_names = {v:k for k,v in kwargs.items()}
+ col_names = {simple_varname(v):k for k,v in kwargs.items()}
+ if None in col_names:
+ raise ValueError("Rename needs column name (e.g. 'a' or _.a), but received %s"%col_names[None])
return __data.rename(columns = col_names)
@rename.register(DataFrameGroupBy)
def _rename(__data, **kwargs):
- raise Exception("Selecting columns of grouped DataFrame currently not allowed")
+ raise NotImplementedError("Selecting columns of grouped DataFrame currently not allowed")
@@ -616,6 +638,11 @@ def arrange(__data, *args):
.drop(tmp_colnames, axis = 1)
+@arrange.register(DataFrameGroupBy)
+def _arrange(__data, *args):
+ raise NotImplementedError("TODO: arrange with grouped DataFrame")
+
+
# Distinct ====================================================================
@@ -890,6 +917,9 @@ def semi_join(left, right = None, on = None):
return left.merge(right.loc[:,on_cols], how = 'inner', on = on_cols)
+@singledispatch2(pd.DataFrame)
+def anti_join(left, right = None, on = None):
+ raise NotImplementedError("anti_join not currently implemented")
left_join = partial(join, how = "left")
right_join = partial(join, how = "right")
diff --git a/siuba/siu.py b/siuba/siu.py
index 3dd4463b..9d5b38e8 100644
--- a/siuba/siu.py
+++ b/siuba/siu.py
@@ -502,6 +502,9 @@ def slice_to_call(x):
return strip_symbolic(x)
+def str_to_getitem_call(x):
+ return Call("__getitem__", MetaArg("_"), x)
+
def strip_symbolic(x):
if isinstance(x, Symbolic):
diff --git a/siuba/sql/dialects/postgresql.py b/siuba/sql/dialects/postgresql.py
index 124e3e01..31b01ed8 100644
--- a/siuba/sql/dialects/postgresql.py
+++ b/siuba/sql/dialects/postgresql.py
@@ -1,14 +1,42 @@
# sqlvariant, allow defining 3 namespaces to override defaults
-from ..translate import base_scalar, base_agg, base_win, SqlTranslator
+from ..translate import (
+ base_scalar, base_agg, base_win, SqlTranslator,
+ win_agg, sql_scalar
+ )
import sqlalchemy.sql.sqltypes as sa_types
from sqlalchemy import sql
+def sql_log(col, base = None):
+ if base is None:
+ return sql.func.ln(col)
+ return sql.func.log(col)
+
def sql_round(col, n):
return sql.func.round(sql.cast(col, sa_types.Numeric()), n)
+def sql_str_contains(col, pat, case, *args, **kwargs):
+ if args or kwargs:
+ raise NotImplementedError("Only pat and case arg of contains allowed.")
+
+ infix = "~" if case else "~*"
+
+ return col.op(infix, pat)
+
+# handle when others is a list?
+def sql_str_cat(col, others, sep, join = None):
+ if join is not None:
+ raise NotImplementedError("join argument of cat not supported")
+
+
scalar = SqlTranslator(
base_scalar,
- round = sql_round
+ log = sql_log,
+ round = sql_round,
+ contains = sql_str_contains,
+ year = lambda col: sql.func.extract('year', sql.cast(col, sql.sqltypes.Date)),
+ concat = sql.func.concat,
+ cat = sql.func.concat,
+ str_c = sql.func.concat
)
aggregate = SqlTranslator(
@@ -16,7 +44,10 @@ def sql_round(col, n):
)
window = SqlTranslator(
- base_win
+ base_win,
+ any = win_agg("bool_or"),
+ all = win_agg("bool_and"),
+ lag = win_agg("lag")
)
funcs = dict(scalar = scalar, aggregate = aggregate, window = window)
diff --git a/siuba/sql/dialects/sqlite.py b/siuba/sql/dialects/sqlite.py
index 910c971c..de30b0e2 100644
--- a/siuba/sql/dialects/sqlite.py
+++ b/siuba/sql/dialects/sqlite.py
@@ -1,5 +1,5 @@
# sqlvariant, allow defining 3 namespaces to override defaults
-from ..translate import base_scalar, base_agg, base_win, SqlTranslator, win_agg
+from ..translate import base_scalar, base_agg, base_nowin, SqlTranslator, win_agg
import sqlalchemy.sql.sqltypes as sa_types
from sqlalchemy import sql
@@ -13,7 +13,7 @@
window = SqlTranslator(
# TODO: should check sqlite version, since < 3.25 can't use windows
- base_win,
+ base_nowin,
sd = win_agg("stddev")
)
diff --git a/siuba/sql/translate.py b/siuba/sql/translate.py
index b36cb4ee..b6930336 100644
--- a/siuba/sql/translate.py
+++ b/siuba/sql/translate.py
@@ -1,7 +1,19 @@
+"""
+This module holds default translations from pandas syntax to sql for 3 kinds of operations...
+
+1. scalar - elementwise operations (e.g. array1 + array2)
+2. aggregation - operations that result in a single number (e.g. array1.mean())
+3. window - operations that do calculations across a window
+ (e.g. array1.lag() or array1.expanding().mean())
+
+
+"""
+
from sqlalchemy import sql
from sqlalchemy.sql import sqltypes as types
from functools import singledispatch
from .verbs import case_when, if_else
+import warnings
# TODO: must make these take both tbl, col as args, since hard to find window funcs
def sa_is_window(clause):
@@ -9,35 +21,80 @@ def sa_is_window(clause):
or isinstance(clause, sql.elements.WithinGroup)
-def sa_modify_window(clause, columns, group_by = None, order_by = None):
- cls = clause.__class__ if sa_is_window(clause) else getattr(clause, "over")
+def sa_modify_window(clause, group_by = None, order_by = None):
if group_by:
- partition_by = [columns[name] for name in group_by]
- return cls(**{**clause.__dict__, 'partition_by': partition_by})
+ group_cols = [columns[name] for name in group_by]
+ partition_by = sql.elements.ClauseList(*group_cols)
+ clone = clause._clone()
+ clone.partition_by = partition_by
+
+ return clone
return clause
+from sqlalchemy.sql.elements import Over
+# windowed agg (group by)
+# agg
+# windowed scalar
+# ordered set agg
+
+class CustomOverClause: pass
+
+class AggOver(Over, CustomOverClause):
+ def set_over(self, group_by, order_by = None):
+ self.partition_by = group_by
+ return self
+
+
+class RankOver(Over, CustomOverClause):
+ def set_over(self, group_by, order_by = None):
+ self.partition_by = group_by
+ return self
+
+
+class CumlOver(Over, CustomOverClause):
+ def set_over(self, group_by, order_by):
+ self.partition_by = group_by
+ self.order_by = order_by
+
+ if not len(order_by):
+ warnings.warn(
+ "No order by columns explicitly set in window function. SQL engine"
+ "does not guarantee a row ordering. Recommend using an arrange beforehand.",
+ RuntimeWarning
+ )
+ return self
+
+
+def win_absent(name):
+ def not_implemented(*args, **kwargs):
+ raise NotImplementedError("SQL dialect does not support {}.".format(name))
+
+ return not_implemented
def win_over(name):
sa_func = getattr(sql.func, name)
- return lambda col: sa_func().over(order_by = col)
+ return lambda col: RankOver(sa_func(), order_by = col)
+def win_cumul(name):
+ sa_func = getattr(sql.func, name)
+ return lambda col: CumlOver(sa_func(col), rows = (None,0))
def win_agg(name):
sa_func = getattr(sql.func, name)
- return lambda col: sa_func(col).over()
+ return lambda col: AggOver(sa_func(col))
def sql_agg(name):
sa_func = getattr(sql.func, name)
return lambda col: sa_func(col)
-def sql_scalar(name):
+def sql_scalar(name, *args):
sa_func = getattr(sql.func, name)
- return lambda col: sa_func(col)
+ return lambda col: sa_func(col, *args)
-def sql_colmeth(meth):
+def sql_colmeth(meth, *outerargs):
def f(col, *args):
- return getattr(col, meth)(*args)
+ return getattr(col, meth)(*outerargs, *args)
return f
def sql_astype(col, _type):
@@ -47,7 +104,10 @@ def sql_astype(col, _type):
float: types.Numeric,
bool: types.Boolean
}
- sa_type = mappings[_type]
+ try:
+ sa_type = mappings[_type]
+ except KeyError:
+ raise ValueError("sql astype currently only supports type objects: str, int, float, bool")
return sql.cast(col, sa_type)
base_scalar = dict(
@@ -70,9 +130,10 @@ def sql_astype(col, _type):
# TODO: I think these are postgres specific?
hour = lambda col: sql.func.date_trunc('hour', col),
week = lambda col: sql.func.date_trunc('week', col),
- isna = lambda col: col.is_(None),
- isnull = lambda col: col.is_(None),
+ isna = sql_colmeth("is_", None),
+ isnull = sql_colmeth("is_", None),
# dply.vector funcs ----
+ desc = lambda col: col.desc(),
# TODO: string methods
#str.len,
@@ -83,7 +144,6 @@ def sql_astype(col, _type):
#str_trim func to cut text off sides
# TODO: move to postgres specific
n = lambda col: sql.func.count(),
- sum = sql_scalar("sum"),
# TODO: this is to support a DictCall (e.g. used in case_when)
dict = dict,
# TODO: don't use singledispatch to add sql support to case_when
@@ -93,13 +153,16 @@ def sql_astype(col, _type):
base_agg = dict(
mean = sql_agg("avg"),
+ sum = sql_agg("sum"),
+ min = sql_agg("min"),
+ max = sql_agg("max"),
# TODO: generalize case where doesn't use col
# need better handeling of vector funcs
len = lambda col: sql.func.count()
)
base_win = dict(
- row_number = win_over("row_number"),
+ row_number = lambda col: CumlOver(sql.func.row_number()),
min_rank = win_over("rank"),
rank = win_over("rank"),
dense_rank = win_over("dense_rank"),
@@ -130,12 +193,47 @@ def sql_astype(col, _type):
# cumulative funcs ---
#avg("id") OVER (PARTITION BY "email" ORDER BY "id" ROWS UNBOUNDED PRECEDING)
#cummean = win_agg("
- #cumsum
+ cumsum = win_cumul("sum")
#cummin
#cummax
)
+# based on https://github.com/tidyverse/dbplyr/blob/master/R/backend-.R
+base_nowin = dict(
+ row_number = win_absent("ROW_NUMBER"),
+ min_rank = win_absent("RANK"),
+ rank = win_absent("RANK"),
+ dense_rank = win_absent("DENSE_RANK"),
+ percent_rank = win_absent("PERCENT_RANK"),
+ cume_dist = win_absent("CUME_DIST"),
+ ntile = win_absent("NTILE"),
+ mean = win_absent("AVG"),
+ sd = win_absent("SD"),
+ var = win_absent("VAR"),
+ cov = win_absent("COV"),
+ cor = win_absent("COR"),
+ sum = win_absent("SUM"),
+ min = win_absent("MIN"),
+ max = win_absent("MAX"),
+ median = win_absent("PERCENTILE_CONT"),
+ quantile = win_absent("PERCENTILE_CONT"),
+ n = win_absent("N"),
+ n_distinct = win_absent("N_DISTINCT"),
+ cummean = win_absent("MEAN"),
+ cumsum = win_absent("SUM"),
+ cummin = win_absent("MIN"),
+ cummax = win_absent("MAX"),
+ nth = win_absent("NTH_VALUE"),
+ first = win_absent("FIRST_VALUE"),
+ last = win_absent("LAST_VALUE"),
+ lead = win_absent("LEAD"),
+ lag = win_absent("LAG"),
+ order_by = win_absent("ORDER_BY"),
+ str_flatten = win_absent("STR_FLATTEN"),
+ count = win_absent("COUNT")
+ )
+
funcs = dict(scalar = base_scalar, aggregate = base_agg, window = base_win)
# MISC ===========================================================================
diff --git a/siuba/sql/verbs.py b/siuba/sql/verbs.py
index 722e380c..c2685182 100644
--- a/siuba/sql/verbs.py
+++ b/siuba/sql/verbs.py
@@ -1,6 +1,6 @@
from siuba.dply.verbs import (
singledispatch2,
- pipe_no_args,
+ show_query, collect,
simple_varname,
select, VarList, var_select,
mutate,
@@ -10,18 +10,18 @@
count,
group_by, ungroup,
case_when,
- join, left_join, right_join, inner_join,
+ join, left_join, right_join, inner_join, semi_join, anti_join,
head,
rename,
distinct,
if_else
)
-from .translate import sa_modify_window, sa_is_window
+from .translate import sa_modify_window, sa_is_window, CustomOverClause
from .utils import get_dialect_funcs
from sqlalchemy import sql
import sqlalchemy
-from siuba.siu import Call, CallTreeLocal
+from siuba.siu import Call, CallTreeLocal, str_to_getitem_call, Lazy
# TODO: currently needed for select, but can we remove pandas?
from pandas import Series
import pandas as pd
@@ -57,17 +57,26 @@ class WindowReplacer(CallListener):
TODO: could replace with a sqlalchemy transformer
"""
- def __init__(self, columns, group_by, window_cte = None):
+ def __init__(self, columns, group_by, order_by, window_cte = None):
self.columns = columns
self.group_by = group_by
+ self.order_by = order_by
self.window_cte = window_cte
self.windows = []
def exit(self, node):
# evaluate
col_expr = node(self.columns)
- if sa_is_window(col_expr):
- label = sa_modify_window(col_expr, self.columns, self.group_by).label(None)
+ if isinstance(col_expr, CustomOverClause):
+ group_by = sql.elements.ClauseList(
+ *[self.columns[name] for name in self.group_by]
+ )
+ order_by = sql.elements.ClauseList(
+ *_create_order_by_clause(self.columns, *self.order_by)
+ )
+
+ label = col_expr.set_over(group_by, order_by).label(None)
+ #label = sa_modify_window(col_expr, self.columns, self.group_by).label(None)
self.windows.append(label)
@@ -81,8 +90,8 @@ def exit(self, node):
return col_expr
-def track_call_windows(call, columns, group_by, window_cte = None):
- listener = WindowReplacer(columns, group_by, window_cte)
+def track_call_windows(call, columns, group_by, order_by, window_cte = None):
+ listener = WindowReplacer(columns, group_by, order_by, window_cte)
col = listener.enter(call)
return col, listener.windows
@@ -93,15 +102,22 @@ def lift_inner_cols(tbl):
return sql.base.ImmutableColumnCollection(data, cols)
-def has_windows(clause):
- windows = []
- append_win = lambda col: windows.append(col)
+def col_expr_requires_cte(call, sel):
+ """Return whether a variable assignment needs a CTE"""
+
+ call_vars = set(call.op_vars(attr_calls = False))
- sql.util.visitors.traverse(clause, {}, {"over": append_win})
- if len(windows):
- return True
+ columns = lift_inner_cols(sel)
+ sel_labs = set(k for k,v in columns.items() if isinstance(v, sql.elements.Label))
+
+ return ( len(sel._group_by_clause)
+ or len(sel._order_by_clause)
+ or not sel_labs.isdisjoint(call_vars)
+ )
- return False
+def get_missing_columns(call, columns):
+ missing_cols = set(call.op_vars(attr_calls = False)) - set(columns.keys())
+ return missing_cols
def compile_el(tbl, el):
compiled = el.compile(
@@ -110,6 +126,14 @@ def compile_el(tbl, el):
)
return compiled
+# Misc utilities --------------------------------------------------------------
+
+def ordered_union(x, y):
+ dx = {el: True for el in x}
+ dy = {el: True for el in y}
+
+ return tuple({**dx, **dy})
+
@@ -140,22 +164,25 @@ def __init__(
self.rm_attr = rm_attr
self.call_sub_attr = call_sub_attr
- def append_op(self, op):
- return self.__class__(
- self.source,
- self.tbl,
- self.ops + [op],
- self.group_by,
- self.order_by,
- self.funcs,
- self.rm_attr,
- self.call_sub_attr
- )
+ def append_op(self, op, **kwargs):
+ cpy = self.copy(**kwargs)
+ cpy.ops = cpy.ops + [op]
+ return cpy
def copy(self, **kwargs):
return self.__class__(**{**self.__dict__, **kwargs})
- def shape_call(self, call, window = True):
+ def shape_call(self, call, window = True, str_accessors = False):
+ # TODO: error if mutate receives a literal value?
+ if str_accessors and isinstance(call, str):
+ # verbs that can use strings as accessors, like group_by, or
+ # arrange, need to convert those strings into a getitem call
+ return str_to_get_item_call(call)
+ elif not isinstance(call, Call):
+ # verbs that use literal strings, need to convert them to a call
+ # that returns a sqlalchemy "literal" object
+ return Lazy(sql.literal(call))
+
f_dict1 = self.funcs['scalar']
f_dict2 = self.funcs['window' if window else 'aggregate']
@@ -172,25 +199,54 @@ def track_call_windows(self, call, columns = None, window_cte = None):
"""Returns tuple of (new column expression, list of window exprs)"""
columns = self.last_op.columns if columns is None else columns
- return track_call_windows(call, columns, self.group_by, window_cte)
+ return track_call_windows(call, columns, self.group_by, self.order_by, window_cte)
+
+ def get_ordered_col_names(self):
+ ungrouped = [k for k in self.last_op.columns.keys() if k not in self.group_by]
+ return list(self.group_by) + ungrouped
@property
def last_op(self):
return self.ops[-1] if len(self.ops) else None
- def __repr__(self):
- tbl_small = self.append_op(self.last_op.limit(5))
-
- # makes sure to get engine, even if sqlalchemy connection obj
- engine = self.source.engine
+ def _get_preview(self):
+ # need to make prev op a cte, so we don't override any previous limit
+ new_sel = sql.select([self.last_op.alias()]).limit(5)
+ tbl_small = self.append_op(new_sel)
+ return collect(tbl_small)
- return ("# Source: lazy query\n"
+ def __repr__(self):
+ template = (
+ "# Source: lazy query\n"
"# DB Conn: {}\n"
"# Preview:\n{}\n"
"# .. may have more rows"
- .format(repr(engine), repr(collect(tbl_small)))
)
+ return template.format(repr(self.source.engine), repr(self._get_preview()))
+
+ def _repr_html_(self):
+ template = (
+ ""
+ "
"
+ "# Source: lazy query\n"
+ "# DB Conn: {}\n"
+ "# Preview:\n"
+ ""
+ "{}"
+ "
# .. may have more rows
"
+ "
(grouped data frame)
" + self._selected_obj._repr_html_() + "