Feature : H3 Index & Pcodes

.dockerignore

@@ -3,3 +3,12 @@
 infra
 docs
 tests
+.env
+.env.*
+*.pyc
+__pycache__
+.pytest_cache
+*.log
+exports/
+.vscode
+.idea

.github/workflows/Unit-Test.yml

@@ -16,7 +16,10 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgis/postgis:14-3.3
+        image: ghcr.io/hotosm/postgis-h3:latest
+        credentials:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
         env:
           POSTGRES_PASSWORD: admin
           POSTGRES_DB: raw
@@ -46,18 +49,29 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v1
         with:
-          python-version: 3.11
+          python-version: 3.12
+      - name: Install uv
+        run: pip install uv
 
-      - name: Insert sample db data
+      - name: Create H3 extensions
         run: |
-          export PGPASSWORD='admin';
-          psql -U postgres -h localhost -p 5434 raw  < tests/fixtures/pokhara.sql
-          psql -U postgres -h localhost -p 5434 raw  < backend/sql/countries.sql
-          psql -U postgres -h localhost -p 5434 raw  < API/data/tables.sql
-          psql -U postgres -h localhost -p 5434 raw  < API/data/cron.sql
+          psql -U postgres -h localhost -p 5434 -d raw -c "CREATE EXTENSION IF NOT EXISTS postgis;"
+          psql -U postgres -h localhost -p 5434 -d raw -c "CREATE EXTENSION IF NOT EXISTS h3;"
+          psql -U postgres -h localhost -p 5434 -d raw -c "CREATE EXTENSION IF NOT EXISTS h3_postgis;"
+
+      - name: Install osm2pgsql for data loading
+        run: sudo apt-get install -y osm2pgsql
+
+      - name: Run Alembic migrations
+        run: |
+          uv sync --group backend
+          uv run alembic upgrade head
+
+      - name: Insert sample test data
+        run: uv run raw-backend --insert --fq m
 
       - name: Install gdal
         run: sudo apt-get update && sudo apt-get -y install gdal-bin python3-gdal && sudo apt-get -y autoremove && sudo apt-get clean
@@ -71,20 +85,14 @@ jobs:
           sudo apt-get install redis
           redis-cli ping
 
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          python setup.py install
-
       - name: Launch ondemand Worker
         run: |
-          celery --app API.api_worker worker --loglevel=INFO --queues='raw_ondemand' &
+          uv run celery --app API.api_worker worker --loglevel=INFO --queues='raw_ondemand' &
 
       - name: Launch daemon Worker
         run: |
-          celery --app API.api_worker worker --loglevel=INFO --queues='raw_daemon' &
+          uv run celery --app API.api_worker worker --loglevel=INFO --queues='raw_daemon' &
 
       - name: Run Tests
         run: |
-          python -m pytest -v -s
+          uv run pytest -v -s

.github/workflows/black-for-PR.yml

@@ -4,7 +4,7 @@ on:
     paths-ignore:
       - 'infra/**'
     branches:
-      - master
+      - stage
       - develop
 jobs:
   linter_name:

.github/workflows/black-formatter.yml

@@ -4,7 +4,7 @@ on:
     paths-ignore:
       - 'infra/**'
     branches:
-      - master
+      - stage
       - develop
 jobs:
   linter_name:

.github/workflows/build-postgres-image.yml

@@ -0,0 +1,34 @@
+name: Build PostgreSQL with H3 Image
+
+on:
+  push:
+    branches:
+      - develop
+      - stage
+    paths:
+      - 'Dockerfile.postgish3'
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Log in to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.postgish3
+          push: true
+          tags: ghcr.io/hotosm/postgis-h3:latest

.github/workflows/build.yml

@@ -24,7 +24,10 @@ jobs:
 
     services:
       postgres:
-        image: postgis/postgis:14-3.3
+        image: ghcr.io/hotosm/postgis-h3:latest
+        credentials:
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
         env:
           POSTGRES_PASSWORD: admin
           POSTGRES_DB: raw
@@ -33,20 +36,27 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 2
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v1
         with:
-          python-version: 3.11
+          python-version: 3.12
+      - name: Install uv
+        run: pip install uv
       - name: Install necessary dependecies for rawdata loading
         run: sudo apt-get update && sudo apt-get install osm2pgsql
       - name: check version
         run: osm2pgsql --version
 
+      - name: Create H3 extensions
+        run: |
+          psql -U postgres -h localhost -p 5434 -d raw -c "CREATE EXTENSION IF NOT EXISTS h3;"
+          psql -U postgres -h localhost -p 5434 -d raw -c "CREATE EXTENSION IF NOT EXISTS h3_postgis;"
+
       - name: Install python requirements for load
-        run: pip install -r backend/requirements.txt
+        run: uv sync --group backend
 
       - name: Start Loading
-        run: python backend/raw_backend --insert --fq m
+        run: uv run raw-backend --insert --fq m
 
       - name: Install gdal
         run: |
@@ -62,10 +72,7 @@ jobs:
           redis-cli ping
 
       - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install -e .
+        run: uv sync
 
       - name: Run uvicorn server
         run: |

.github/workflows/code-check.yml

@@ -4,11 +4,11 @@ on:
     paths-ignore:
       - "infra/**"
     branches:
-      - master
+      - stage
       - develop
   pull_request:
     branches:
-      - master
+      - stage
       - develop
   workflow_dispatch:
 jobs:
@@ -23,8 +23,8 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install flake8
-          pip install flake8-docstrings
+          pip install uv
+          uv pip install flake8 flake8-docstrings
       - name: Lint with flake8 for src
         run: |
           flake8 src --count --select=E9,F63,F7,F82,D102 --docstring-convention numpy --show-source --statistics --ignore=E501,C901

.github/workflows/docker-image-build.yml

@@ -3,12 +3,12 @@ name: Docker Image CI
 on:
   push:
     branches:
-      - master
+      - stage
       - develop
 
   pull_request:
     branches:
-      - master
+      - stage
       - develop
 jobs:
   build:

.github/workflows/publish_mkdocs.yml

@@ -3,7 +3,7 @@ on:
   push:
     branches:
       - develop
-      - master
+      - stage
     paths:
       # Only rebuild documentation when docs have changed
       - "docs/**"
@@ -22,13 +22,14 @@ jobs:
           publish_branch: gh-pages
       - name: Install Dependencies
         run: |
-          pip install -r requirements.txt
-          pip install mkdocs-material==8.5.11 mkdocs-jupyter==0.22.0 neoteroi-mkdocs==0.1.2 pdocs==1.0.1 lxml_html_clean==0.1.1
-          python setup.py install
+          pip install uv
+          uv sync
+          uv pip install mkdocs-material==8.5.11 mkdocs-jupyter==0.22.0 neoteroi-mkdocs==0.1.2 pdocs==1.0.1 lxml_html_clean==0.1.1
+          uv run python setup.py install
           mv config.txt.sample config.txt
 
       - name: Update API docs
         run: |
-          pdocs as_markdown -o docs/src/api --exclude_source --overwrite src API
+          uv run pdocs as_markdown -o docs/src/api --exclude_source --overwrite src API
       - name: Deploy Documentation
-        run: python -m mkdocs  gh-deploy --force -f docs/mkdocs.yml
+        run: uv run python -m mkdocs  gh-deploy --force -f docs/mkdocs.yml

.python-version

@@ -0,0 +1 @@
+3.13

API/api_worker.py

@@ -15,11 +15,10 @@
 from celery import Celery
 
 # Reader imports
-from src.app import CustomExport, PolygonStats, RawData, S3FileTransfer
-from src.config import ALLOW_BIND_ZIP_FILTER, CELERY_BROKER_HEARTBEAT
-from src.config import CELERY_BROKER_URL as celery_broker_uri
-from src.config import CELERY_RESULT_BACKEND as celery_backend
+from src.app import CustomExport, RawData, S3FileTransfer
 from src.config import (
+    ALLOW_BIND_ZIP_FILTER,
+    CELERY_BROKER_HEARTBEAT,
     CELERY_WORKER_LOST_WAIT,
     DEFAULT_HARD_TASK_LIMIT,
     DEFAULT_README_TEXT,
@@ -29,9 +28,11 @@
     EXPORT_PATH,
     HDX_HARD_TASK_LIMIT,
     HDX_SOFT_TASK_LIMIT,
+    WORKER_PREFETCH_MULTIPLIER,
 )
+from src.config import CELERY_BROKER_URL as celery_broker_uri
+from src.config import CELERY_RESULT_BACKEND as celery_backend
 from src.config import USE_S3_TO_UPLOAD as use_s3_to_upload
-from src.config import WORKER_PREFETCH_MULTIPLIER
 from src.config import logger as logging
 from src.query_builder.builder import format_file_name_str
 from src.validation.models import (
@@ -231,14 +232,14 @@ def process_raw_data(self, params, user=None):
         ).extract_current_data(file_parts)
         inside_file_size = 0
         polygon_stats = None
-        if "include_stats" in params.dict():
-            if params.include_stats:
-                feature = {
-                    "type": "Feature",
-                    "geometry": json.loads(params.geometry.model_dump_json()),
-                    "properties": {},
-                }
-                polygon_stats = PolygonStats(feature).get_summary_stats()
+        # if "include_stats" in params.dict():
+        #     if params.include_stats:
+        #         feature = {
+        #             "type": "Feature",
+        #             "geometry": json.loads(params.geometry.model_dump_json()),
+        #             "properties": {},
+        #         }
+        #         # polygon_stats = PolygonStats(feature).get_summary_stats()
         if bind_zip:
             upload_file_path, inside_file_size = zip_binding(
                 working_dir=working_dir,