Dockerfile

.github/workflows/docker.yml

@@ -0,0 +1,23 @@
+name: docker-image
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Login to Docker Hub
+      uses: docker/login-action@v1
+      with:
+        username: ${{ secrets.DOCKER_USERNAME }}
+        password: ${{ secrets.DOCKER_PASSWORD }}
+    - name: Build Docker image
+      run: |
+        docker build . -t hiprimus/optimus:latest
+    - name: Push Docker image
+      run: |
+        docker push hiprimus/optimus:latest

.github/workflows/main.yml

@@ -0,0 +1,36 @@
+name: tests
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+  pull_request:
+    branches: 
+      - main
+      - develop-23.5
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8]
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements/pandas-requirements.txt ]; then pip install -r requirements/pandas-requirements.txt; fi
+          if [ -f requirements/dask-requirements.txt ]; then pip install -r requirements/dask-requirements.txt; fi
+          if [ -f requirements/api-requirements.txt ]; then pip install -r requirements/api-requirements.txt; fi
+          python -m nltk.downloader punkt wordnet stopwords averaged_perceptron_tagger omw-1.4
+      - name: Test with unittest
+        run: |
+          python -m unittest

.github/workflows/python-manually-publish-test.yml

@@ -0,0 +1,67 @@
+name: Update Version and Publish to Test PyPI
+
+on:
+  workflow_dispatch:
+    inputs:
+      new_version:
+        description: 'New PyOptimus version (TestPyPI)'
+        required: true
+      python_version:
+        description: 'Python version'
+        default: '3.10'
+        required: true
+
+jobs:
+  update-version-and-create-tag:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python ${{ github.event.inputs.python_version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: '${{ github.event.inputs.python_version }}'
+
+      - name: Update version in toml file
+        run: |
+          sed -i 's/version\s*=\s*".*"/version = "${{ github.event.inputs.new_version }}"/' pyproject.toml
+
+      - name: Commit changes
+        run: |
+          git config --local user.email "${{ github.actor }}@users.noreply.github.com"
+          git config --local user.name "${{ github.actor }}"
+          git commit -m "Update version in toml file" pyproject.toml
+
+      - name: Push changes
+        uses: ad-m/github-push-action@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          branch: ${{ github.ref }}
+
+      - name: Create tag
+        run: |
+          git tag "test-${{ github.event.inputs.new_version }}"
+
+      - name: Push tag
+        run: |
+          git push origin "test-${{ github.event.inputs.new_version }}"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+
+      - name: Configure Poetry
+        run: |
+          poetry config repositories.pyoptimus https://test.pypi.org/legacy/
+          poetry config http-basic.pyoptimus ${{ secrets.TEST_PYPI_USERNAME }} ${{ secrets.TEST_PYPI_PASSWORD }}
+
+      - name: Build and publish package
+        run: |
+          poetry build
+          poetry publish -r pyoptimus
+        env:
+          TEST_PYPI_USERNAME: ${{ secrets.TEST_PYPI_USERNAME }}
+          TEST_PYPI_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }}

.github/workflows/python-publish.yml

@@ -0,0 +1,29 @@
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install wheel
+        pip install build
+    - name: Build package
+      run: python setup.py sdist bdist_wheel --no-git
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        user: ${{ secrets.PYPI_USERNAME }}
+        password: ${{ secrets.PYPI_TOKEN }}

.gitignore

@@ -5,13 +5,12 @@
 # Directory-based project format:
 .idea/
 
-# Optimus
-
-examples/checkPointFolder
+# Visual Studio Code:
+.vscode/
 
+# Optimus
 metastore_db
 spark-warehouse
-projectFilesBackup/tests.py
 
 # Distribution / packaging
 build/
@@ -28,39 +27,12 @@ __pycache__
 .coverage*
 .DS_Store
 .pytest_cache/
-data.json
-.pytest_cache/README.md
-examples/new-api-jar-test\.ipynb
 venv/
 
-# Test
-tests/test.csv/
-tests/test.json/
-tests/test.parquet/
 *.test
 
-examples/new-api-optimus-jars\.ipynb
-readme/credentials\.py
-readme/data/foo\.json/
-readme/data/foo\.parquet/
-readme/data/foo\.csv/
-
-examples/test\.parquet/
-examples/test\.csv/
-examples/test\.json/
-examples/temp
-
-
-node_modules
-
-
-*bumblebee\.ini
-examples/many-columns\.csv
-
-examples/many=columns\.parquet/
-
-examples/many-columns.parquet/
-
-examples/sandbox.ipynb
-
-examples/dask-worker-space/
+node_modules/
+**/dask-worker-space/
+dask-worker-space/
+conda/recipes/conda
+conda/recipes/conda-build

.pyup.yml

@@ -0,0 +1,3 @@
+update: insecure
+branch: develop-23.5
+schedule: "every two weeks on monday"

.readthedocs.yaml

@@ -0,0 +1,21 @@
+# ..readthedocs.yaml
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+# Build documentation in the docs/ directory with Sphinx
+sphinx:
+   configuration: docs/source/conf.py
+
+# Optionally build your docs in additional formats such as PDF
+formats:
+   - pdf
+
+# Optionally set the version of Python and requirements required to build your docs
+python:
+   version: "3.7"
+   install:
+     - requirements: requirements.txt
+     - requirements: docs/requirements.txt

CODE_OF_CONDUCT.md

@@ -40,7 +40,7 @@ Project maintainers who do not follow or enforce the Code of Conduct in good fai
 
 ## Developer & Administrator
 
-[Favio Vázquez](https://github.com/faviovazquez)
+[Argenis Leon](https://github.com/argenisleon)
 
 ## Attribution
 

CONTRIBUTING.md

@@ -19,7 +19,7 @@ Please note we have a code of conduct, please follow it in all your interactions
 
 ### Developer and administrator
 
-[Favio Vázquez](https://github.com/faviovazquez)
+[Argenis León](https://github.com/argenisleon)
 
 ### Attribution
 

Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:20.04
+
+RUN apt-get update && yes|apt-get upgrade && \
+    apt-get install -y git curl wget nano bzip2 sudo net-tools && \
+    apt-get install -y --no-install-recommends apt-utils
+
+RUN wget https://repo.anaconda.com/archive/Anaconda3-2020.07-Linux-x86_64.sh && \
+    bash Anaconda3-2020.07-Linux-x86_64.sh -b && \
+    rm Anaconda3-2020.07-Linux-x86_64.sh
+
+ENV PATH="/root/anaconda3/bin:${PATH}"
+
+RUN sudo apt-get update --fix-missing && \
+    sudo apt-get install -y gcc g++ && \
+    sudo apt-get clean
+
+RUN sudo rm -rf /var/lib/apt/lists/*
+
+RUN sudo chown -R root ~/anaconda3/bin && \
+    sudo chmod -R +x ~/anaconda3/bin && \
+    conda install -c conda-forge jupyterlab && \
+    conda install -c conda-forge dask-labextension && \
+    jupyter serverextension enable dask_labextension && \
+    conda install -c conda-forge jupyter_kernel_gateway && \
+    conda clean -afy
+
+RUN echo "Version 23.5.0-beta"
+
+RUN pip install cytoolz && \
+    pip install llvmlite --ignore-installed && \
+    pip install git+https://github.com/hi-primus/[email protected]#egg=pyoptimus[pandas] && \
+    pip install git+https://github.com/hi-primus/[email protected]#egg=pyoptimus[dask]
+
+CMD jupyter notebook --port=8888 --no-browser --ip=0.0.0.0 --allow-root
+
+EXPOSE 8888:8888 8889:8889

Dockerfile.coiled

@@ -0,0 +1,28 @@
+FROM continuumio/miniconda3:4.9.2
+
+RUN apt-get update --fix-missing && \
+    apt-get install -y gcc && \
+    apt-get install -y g++ && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN conda install libmagic && \
+    conda clean -afy
+
+RUN pip install aiohttp --no-cache-dir && \
+    pip install git+https://github.com/hi-primus/[email protected] --no-cache-dir
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+