From be916f58bd2b9f7fd1cbb40327fdb4c9faed6df8 Mon Sep 17 00:00:00 2001 From: Bas Harenslak Date: Sun, 18 Oct 2020 18:58:26 +0200 Subject: [PATCH 1/2] Rename CH9 to CH7 --- .../{chapter9 => chapter07}/.airflowignore | 0 chapters/chapter07/README.md | 23 +++++++++++++++++++ .../dags/custom/__init__.py | 0 .../dags/custom/postgres_to_s3_operator.py | 0 .../dags/listing_7_13_and_14.py} | 2 +- .../dags/listing_7_2.py} | 2 +- .../number-classifier/.chalice/config.json | 0 .../number-classifier/.gitignore | 0 .../digit_classifier/number-classifier/app.py | 0 .../number-classifier/requirements.txt | 0 .../docker-compose-insideairbnb.yml | 0 .../docker-compose.yml | 0 .../insideairbnb/insideairbnb/Dockerfile | 0 .../insideairbnb/postgres-init.sh | 0 .../insideairbnb/numbercruncher/Dockerfile | 0 .../insideairbnb/numbercruncher/crunchdata.py | 0 chapters/chapter9/readme.md | 21 ----------------- 17 files changed, 25 insertions(+), 23 deletions(-) rename chapters/{chapter9 => chapter07}/.airflowignore (100%) create mode 100644 chapters/chapter07/README.md rename chapters/{chapter9 => chapter07}/dags/custom/__init__.py (100%) rename chapters/{chapter9 => chapter07}/dags/custom/postgres_to_s3_operator.py (100%) rename chapters/{chapter9/dags/chapter9_insideairbnb.py => chapter07/dags/listing_7_13_and_14.py} (96%) rename chapters/{chapter9/dags/chapter9_digit_classifier.py => chapter07/dags/listing_7_2.py} (98%) rename chapters/{chapter9 => chapter07}/digit_classifier/number-classifier/.chalice/config.json (100%) rename chapters/{chapter9 => chapter07}/digit_classifier/number-classifier/.gitignore (100%) rename chapters/{chapter9 => chapter07}/digit_classifier/number-classifier/app.py (100%) rename chapters/{chapter9 => chapter07}/digit_classifier/number-classifier/requirements.txt (100%) rename chapters/{chapter9 => chapter07}/docker-compose-insideairbnb.yml (100%) rename chapters/{chapter9 => chapter07}/docker-compose.yml (100%) rename chapters/{chapter9 => chapter07}/insideairbnb/insideairbnb/Dockerfile (100%) rename chapters/{chapter9 => chapter07}/insideairbnb/insideairbnb/postgres-init.sh (100%) rename chapters/{chapter9 => chapter07}/insideairbnb/numbercruncher/Dockerfile (100%) rename chapters/{chapter9 => chapter07}/insideairbnb/numbercruncher/crunchdata.py (100%) delete mode 100644 chapters/chapter9/readme.md diff --git a/chapters/chapter9/.airflowignore b/chapters/chapter07/.airflowignore similarity index 100% rename from chapters/chapter9/.airflowignore rename to chapters/chapter07/.airflowignore diff --git a/chapters/chapter07/README.md b/chapters/chapter07/README.md new file mode 100644 index 00000000..9f68e644 --- /dev/null +++ b/chapters/chapter07/README.md @@ -0,0 +1,23 @@ +# Chapter 7 + +Code accompanying Chapter 7 of the book [Data Pipelines with Apache Airflow](https://www.manning.com/books/data-pipelines-with-apache-airflow). + +## Contents + + +## Usage + +To get started with the code examples, start Airflow with Docker Compose with the following command: + +```bash +docker-compose up -d +``` + +The webserver initializes a few things, so wait for a few seconds, and you should be able to access the +Airflow webserver at http://localhost:8080. + +To stop running the examples, run the following command: + +```bash +docker-compose down -v +``` diff --git a/chapters/chapter9/dags/custom/__init__.py b/chapters/chapter07/dags/custom/__init__.py similarity index 100% rename from chapters/chapter9/dags/custom/__init__.py rename to chapters/chapter07/dags/custom/__init__.py diff --git a/chapters/chapter9/dags/custom/postgres_to_s3_operator.py b/chapters/chapter07/dags/custom/postgres_to_s3_operator.py similarity index 100% rename from chapters/chapter9/dags/custom/postgres_to_s3_operator.py rename to chapters/chapter07/dags/custom/postgres_to_s3_operator.py diff --git a/chapters/chapter9/dags/chapter9_insideairbnb.py b/chapters/chapter07/dags/listing_7_13_and_14.py similarity index 96% rename from chapters/chapter9/dags/chapter9_insideairbnb.py rename to chapters/chapter07/dags/listing_7_13_and_14.py index 2fa45cc2..c239a365 100644 --- a/chapters/chapter9/dags/chapter9_insideairbnb.py +++ b/chapters/chapter07/dags/listing_7_13_and_14.py @@ -5,7 +5,7 @@ from custom.postgres_to_s3_operator import PostgresToS3Operator dag = DAG( - dag_id="chapter9_insideairbnb", + dag_id="listing_7_13_and_14", start_date=datetime(2015, 4, 5), end_date=datetime(2019, 12, 7), schedule_interval="@monthly", diff --git a/chapters/chapter9/dags/chapter9_digit_classifier.py b/chapters/chapter07/dags/listing_7_2.py similarity index 98% rename from chapters/chapter9/dags/chapter9_digit_classifier.py rename to chapters/chapter07/dags/listing_7_2.py index 53938529..785bb9e8 100644 --- a/chapters/chapter9/dags/chapter9_digit_classifier.py +++ b/chapters/chapter07/dags/listing_7_2.py @@ -16,7 +16,7 @@ from sagemaker.amazon.common import write_numpy_to_dense_tensor dag = DAG( - dag_id="chapter9_aws_handwritten_digit_classifier", + dag_id="listing_7_2", schedule_interval=None, start_date=airflow.utils.dates.days_ago(3), ) diff --git a/chapters/chapter9/digit_classifier/number-classifier/.chalice/config.json b/chapters/chapter07/digit_classifier/number-classifier/.chalice/config.json similarity index 100% rename from chapters/chapter9/digit_classifier/number-classifier/.chalice/config.json rename to chapters/chapter07/digit_classifier/number-classifier/.chalice/config.json diff --git a/chapters/chapter9/digit_classifier/number-classifier/.gitignore b/chapters/chapter07/digit_classifier/number-classifier/.gitignore similarity index 100% rename from chapters/chapter9/digit_classifier/number-classifier/.gitignore rename to chapters/chapter07/digit_classifier/number-classifier/.gitignore diff --git a/chapters/chapter9/digit_classifier/number-classifier/app.py b/chapters/chapter07/digit_classifier/number-classifier/app.py similarity index 100% rename from chapters/chapter9/digit_classifier/number-classifier/app.py rename to chapters/chapter07/digit_classifier/number-classifier/app.py diff --git a/chapters/chapter9/digit_classifier/number-classifier/requirements.txt b/chapters/chapter07/digit_classifier/number-classifier/requirements.txt similarity index 100% rename from chapters/chapter9/digit_classifier/number-classifier/requirements.txt rename to chapters/chapter07/digit_classifier/number-classifier/requirements.txt diff --git a/chapters/chapter9/docker-compose-insideairbnb.yml b/chapters/chapter07/docker-compose-insideairbnb.yml similarity index 100% rename from chapters/chapter9/docker-compose-insideairbnb.yml rename to chapters/chapter07/docker-compose-insideairbnb.yml diff --git a/chapters/chapter9/docker-compose.yml b/chapters/chapter07/docker-compose.yml similarity index 100% rename from chapters/chapter9/docker-compose.yml rename to chapters/chapter07/docker-compose.yml diff --git a/chapters/chapter9/insideairbnb/insideairbnb/Dockerfile b/chapters/chapter07/insideairbnb/insideairbnb/Dockerfile similarity index 100% rename from chapters/chapter9/insideairbnb/insideairbnb/Dockerfile rename to chapters/chapter07/insideairbnb/insideairbnb/Dockerfile diff --git a/chapters/chapter9/insideairbnb/insideairbnb/postgres-init.sh b/chapters/chapter07/insideairbnb/insideairbnb/postgres-init.sh similarity index 100% rename from chapters/chapter9/insideairbnb/insideairbnb/postgres-init.sh rename to chapters/chapter07/insideairbnb/insideairbnb/postgres-init.sh diff --git a/chapters/chapter9/insideairbnb/numbercruncher/Dockerfile b/chapters/chapter07/insideairbnb/numbercruncher/Dockerfile similarity index 100% rename from chapters/chapter9/insideairbnb/numbercruncher/Dockerfile rename to chapters/chapter07/insideairbnb/numbercruncher/Dockerfile diff --git a/chapters/chapter9/insideairbnb/numbercruncher/crunchdata.py b/chapters/chapter07/insideairbnb/numbercruncher/crunchdata.py similarity index 100% rename from chapters/chapter9/insideairbnb/numbercruncher/crunchdata.py rename to chapters/chapter07/insideairbnb/numbercruncher/crunchdata.py diff --git a/chapters/chapter9/readme.md b/chapters/chapter9/readme.md deleted file mode 100644 index 7d59f308..00000000 --- a/chapters/chapter9/readme.md +++ /dev/null @@ -1,21 +0,0 @@ -# Chapter 9 - -Code accompanying Chapter 9 of the book 'Data pipelines with Apache Airflow'. - -## Contents - -This code example contains the following DAGs: - -- chapter9 - Small DAG illustrating the postgres-to-s3 operator. - -## Usage - -To get started with the code examples, start Airflow in docker using the following command: - - docker-compose up -d --build - -Wait for a few seconds and you should be able to access the examples at http://localhost:8080/. - -To stop running the examples, run the following command: - - docker-compose down From eef9a90d2619685aed238a9da4fb324a572a15e7 Mon Sep 17 00:00:00 2001 From: Bas Harenslak Date: Sun, 18 Oct 2020 19:14:24 +0200 Subject: [PATCH 2/2] Rename DAG files --- ...ing_7_2.py => chapter_7_aws_handwritten_digit_classifier.py} | 2 +- .../dags/{listing_7_13_and_14.py => chapter_7_insideairbnb.py} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename chapters/chapter07/dags/{listing_7_2.py => chapter_7_aws_handwritten_digit_classifier.py} (98%) rename chapters/chapter07/dags/{listing_7_13_and_14.py => chapter_7_insideairbnb.py} (96%) diff --git a/chapters/chapter07/dags/listing_7_2.py b/chapters/chapter07/dags/chapter_7_aws_handwritten_digit_classifier.py similarity index 98% rename from chapters/chapter07/dags/listing_7_2.py rename to chapters/chapter07/dags/chapter_7_aws_handwritten_digit_classifier.py index 785bb9e8..77fc60eb 100644 --- a/chapters/chapter07/dags/listing_7_2.py +++ b/chapters/chapter07/dags/chapter_7_aws_handwritten_digit_classifier.py @@ -16,7 +16,7 @@ from sagemaker.amazon.common import write_numpy_to_dense_tensor dag = DAG( - dag_id="listing_7_2", + dag_id="chapter_7_aws_handwritten_digit_classifier", schedule_interval=None, start_date=airflow.utils.dates.days_ago(3), ) diff --git a/chapters/chapter07/dags/listing_7_13_and_14.py b/chapters/chapter07/dags/chapter_7_insideairbnb.py similarity index 96% rename from chapters/chapter07/dags/listing_7_13_and_14.py rename to chapters/chapter07/dags/chapter_7_insideairbnb.py index c239a365..0ccd644f 100644 --- a/chapters/chapter07/dags/listing_7_13_and_14.py +++ b/chapters/chapter07/dags/chapter_7_insideairbnb.py @@ -5,7 +5,7 @@ from custom.postgres_to_s3_operator import PostgresToS3Operator dag = DAG( - dag_id="listing_7_13_and_14", + dag_id="chapter_7_insideairbnb", start_date=datetime(2015, 4, 5), end_date=datetime(2019, 12, 7), schedule_interval="@monthly",