added same code for QFS

DIYBigData · Oct 27, 2019 · 3d1988c · 3d1988c
1 parent fd12efd
commit 3d1988c
Showing 1 changed file with 33 additions and 6 deletions.
diff --git a/...tebooks/prime-numbers/prime-numbers.ipynb → ...oks/prime-numbers/prime-numbers-qfs.ipynb b/...tebooks/prime-numbers/prime-numbers.ipynb → ...oks/prime-numbers/prime-numbers-qfs.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -17,16 +17,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "MAX_VALUE = 1000000000"
+    "MAX_VALUE = 10000000"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -69,7 +69,34 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "values.filter(F.col('is_prime')).count()"
+    "values.repartition(50).write.parquet('qfs:///test/prime-numbers', mode='overwrite')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = spark.read.parquet('qfs:///test/prime-numbers')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.count()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df.filter(F.col('is_prime')).count()"
    ]
   },
   {
@@ -78,7 +105,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "values.filter(F.col('is_prime')).agg(F.max('value')).collect()"
+    "df.filter(F.col('is_prime')).agg(F.max('value').alias('max_prime')).collect()[0].max_prime"
    ]
   },
   {