Do not deliver Spark dependency

* Remove explicit Spark dependency as we might have different Spark version in the environment using the utils
* Run tests against Spark 2.4.4
* Make sure tests are run with Python 3.7, as Spark 2.4 does not support newer Python versions

Makefile

@@ -1,11 +1,16 @@
-.PHONY: setup test
+.PHONY: setup-environment test
+PYTHON=$(shell brew --prefix)/opt/[email protected]/bin/python3
+
+install-python:
+	brew install [email protected]
 
 setup-environment:
-	pip3 install virtualenv==20.0.15
-	virtualenv env
-	source env/bin/activate; \
-	pip3 install -r requirements.txt
+	$(PYTHON) -m pip install virtualenv==20.0.15
+	$(PYTHON) -m virtualenv env
+	. env/bin/activate; \
+	pip3 install -r requirements.txt; \
+	pip3 install pyspark==2.4.4
 
-test:
-	source env/bin/activate; \
+test: setup-environment
+	. env/bin/activate; \
 	SPARK_LOCAL_IP=127.0.0.1 PYTHONPATH=./src python -m pytest $(target) -v

README.md

@@ -29,6 +29,12 @@ def main():
     Job(__package__).execute()
 ```
 
+## How to test
+
+The tests currently run against Spark 2.4, which does not work with Python versions > 3.7 ([reference](https://issues.apache.org/jira/browse/SPARK-29536)). For that reason we expect Python 3.7 to be installed with Homebrew.
+To run the tests, first install Python 3.7 with Homebrew (`make install-python`), then run the tests with `make test`. If you're not using Homebrew you can override the `PYTHON` variable as such: `make test PYTHON=/path/to/python3`.
+
+
 ## How to contribute
 
 If you want to contribute, please submit a Pull Request! Thank you :)

requirements.txt

@@ -1,4 +1,3 @@
-pyspark==3.0.1
 pytest==6.1.2
 pytest-spark==0.6.0
 testfixtures==6.15.0

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="pyspark-core-utils",
-    version="1.0.0",
+    version="1.0.1",
     author="ImmobilienScout24",
     description="PySpark core utils library",
     long_description=long_description,