Skip to content

Commit 7fa7641

Browse files
committed
adding note on fastparquet
1 parent bfeb2ed commit 7fa7641

File tree

1 file changed

+16
-15
lines changed

1 file changed

+16
-15
lines changed

data-wrangling/parquet-from-oss.ipynb

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "markdown",
5-
"id": "aa2046de",
5+
"id": "0da99b91",
66
"metadata": {},
77
"source": [
88
"## Reading/Writing Parquet Files From/to OCI Object Storage with Pandas\n",
@@ -19,7 +19,7 @@
1919
{
2020
"cell_type": "code",
2121
"execution_count": null,
22-
"id": "7b6c2fbc",
22+
"id": "22dc245b",
2323
"metadata": {},
2424
"outputs": [],
2525
"source": [
@@ -29,7 +29,7 @@
2929
{
3030
"cell_type": "code",
3131
"execution_count": null,
32-
"id": "202379bb",
32+
"id": "bd62ebae",
3333
"metadata": {},
3434
"outputs": [],
3535
"source": [
@@ -47,7 +47,7 @@
4747
{
4848
"cell_type": "code",
4949
"execution_count": null,
50-
"id": "55e50263",
50+
"id": "ba1c87b3",
5151
"metadata": {},
5252
"outputs": [],
5353
"source": [
@@ -59,7 +59,7 @@
5959
},
6060
{
6161
"cell_type": "markdown",
62-
"id": "6ab82d93",
62+
"id": "5f68024c",
6363
"metadata": {},
6464
"source": [
6565
"# Single Large File"
@@ -68,7 +68,7 @@
6868
{
6969
"cell_type": "code",
7070
"execution_count": null,
71-
"id": "10759d13",
71+
"id": "2a2d5c4a",
7272
"metadata": {},
7373
"outputs": [],
7474
"source": [
@@ -79,10 +79,11 @@
7979
{
8080
"cell_type": "code",
8181
"execution_count": null,
82-
"id": "f0fdfa93",
82+
"id": "a1b0b709",
8383
"metadata": {},
8484
"outputs": [],
8585
"source": [
86+
"# using the `pyarrow` engine. You can also use `fastparquet`. \n",
8687
"for f in large_files: \n",
8788
" df = pd.read_parquet(f\"oci://{bucket}@{namespace}/{f}\", \n",
8889
" storage_options=default_signer(),\n",
@@ -94,7 +95,7 @@
9495
},
9596
{
9697
"cell_type": "markdown",
97-
"id": "1d1c2c0c",
98+
"id": "06706b05",
9899
"metadata": {},
99100
"source": [
100101
"# Multiple Large Files"
@@ -103,7 +104,7 @@
103104
{
104105
"cell_type": "code",
105106
"execution_count": null,
106-
"id": "51fade09",
107+
"id": "c2d202b5",
107108
"metadata": {},
108109
"outputs": [],
109110
"source": [
@@ -113,7 +114,7 @@
113114
{
114115
"cell_type": "code",
115116
"execution_count": null,
116-
"id": "efb5f2d4",
117+
"id": "8436ded0",
117118
"metadata": {},
118119
"outputs": [],
119120
"source": [
@@ -124,7 +125,7 @@
124125
{
125126
"cell_type": "code",
126127
"execution_count": null,
127-
"id": "d904ade7",
128+
"id": "0e146024",
128129
"metadata": {},
129130
"outputs": [],
130131
"source": [
@@ -135,7 +136,7 @@
135136
{
136137
"cell_type": "code",
137138
"execution_count": null,
138-
"id": "d3c597c4",
139+
"id": "22421b16",
139140
"metadata": {},
140141
"outputs": [],
141142
"source": [
@@ -148,7 +149,7 @@
148149
{
149150
"cell_type": "code",
150151
"execution_count": null,
151-
"id": "cf98a539",
152+
"id": "a27c6c4b",
152153
"metadata": {},
153154
"outputs": [],
154155
"source": [
@@ -158,7 +159,7 @@
158159
},
159160
{
160161
"cell_type": "markdown",
161-
"id": "352c80fe",
162+
"id": "6a52a973",
162163
"metadata": {},
163164
"source": [
164165
"## Write Parquet Files to Object Storage: "
@@ -167,7 +168,7 @@
167168
{
168169
"cell_type": "code",
169170
"execution_count": null,
170-
"id": "795d3503",
171+
"id": "f4923c6b",
171172
"metadata": {},
172173
"outputs": [],
173174
"source": [

0 commit comments

Comments
 (0)