[Done] Upgrade spatialite file version to version 4 (#180)

.dockerignore

@@ -0,0 +1,2 @@
+**
+!requirements*.txt

.github/workflows/test.yml

@@ -54,11 +54,11 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python }}
-      
+
       - name: Install psql, sqlite3 and spatialite
         run: |
           sudo apt-get install --yes --no-install-recommends postgresql-client sqlite3 libsqlite3-mod-spatialite
-      
+  
       - name: Load test postgis database
         shell: bash
         run: | 

CHANGES.rst

@@ -5,7 +5,8 @@ Changelog of threedi-modelchecker
 0.26.2 (unreleased)
 -------------------
 
-- Nothing changed yet.
+- Added ModelSchema().upgrade_spatialite_version (and the same argument to .upgrade) to
+  upgrade the spatialite version from 3 to 4/5.
 
 
 0.26.1 (2022-04-11)

Docker/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.7
+FROM python:3.9-slim-bullseye
+# The bullseye image contains libspatialite 5
 
 RUN apt-get update \
     && apt-get install -y \

docker-compose.yml

@@ -10,8 +10,6 @@ services:
     volumes:
       - .:/code
     working_dir: /code
-    depends_on:
-      - postgis
     command: bash
 
   postgis:

requirements-dev.txt

@@ -1,9 +1,9 @@
-coverage
 factory_boy
-flake8
 pytest
-pytest-cov
-pytest-flake8
 mock
-mypy
-sqlalchemy-stubs
+pytest-cov
+threedi-api-client
+aiofiles
+aiohttp
+pytest-asyncio
+numpy

requirements.txt

@@ -1,5 +1,5 @@
-GeoAlchemy2==0.6.1
-SQLAlchemy==1.3.1
+Click
+GeoAlchemy2>=0.9,!=0.11.*
+SQLAlchemy>=1.2
 Click==7.0
-psycopg2
-alembic
+alembic>=0.9

tests/conftest.py

@@ -89,6 +89,22 @@ def in_memory_sqlite():
     return ThreediDatabase({"db_path": ""}, db_type="spatialite", echo=False)
 
 
+@pytest.fixture
+def empty_sqlite_v3(tmp_path):
+    """An empty spatialite v3 in the latest migration state"""
+    tmp_sqlite = tmp_path / "empty_v3.sqlite"
+    shutil.copyfile(os.path.join(data_dir, "empty_v3.sqlite"), tmp_sqlite)
+    return ThreediDatabase({"db_path": tmp_sqlite}, db_type="spatialite", echo=False)
+
+
+@pytest.fixture
+def empty_sqlite_v3_clone(tmp_path):
+    """An empty spatialite v3 in the latest migration state"""
+    tmp_sqlite = tmp_path / "empty_v3_clone.sqlite"
+    shutil.copyfile(os.path.join(data_dir, "empty_v3.sqlite"), tmp_sqlite)
+    return ThreediDatabase({"db_path": tmp_sqlite}, db_type="spatialite", echo=False)
+
+
 @pytest.fixture
 def south_latest_sqlite(tmp_path):
     """An empty SQLite that is in its latest South migration state"""

tests/test_schema.py

@@ -8,6 +8,7 @@
 from threedi_modelchecker.schema import get_schema_version
 from threedi_modelchecker.schema import ModelSchema
 from threedi_modelchecker.threedi_model.views import ALL_VIEWS
+from threedi_modelchecker.spatialite_versions import get_spatialite_version
 from unittest import mock
 
 import pytest
@@ -86,9 +87,7 @@ def test_get_version_alembic(in_memory_sqlite, alembic_version_table):
 def test_validate_schema(threedi_db):
     """Validate a correct schema version"""
     schema = ModelSchema(threedi_db)
-    with mock.patch.object(
-        schema, "get_version", return_value=get_schema_version()
-    ):
+    with mock.patch.object(schema, "get_version", return_value=get_schema_version()):
         assert schema.validate_schema()
 
 
@@ -113,23 +112,23 @@ def test_validate_schema_too_high_migration(threedi_db, version):
 def test_full_upgrade_empty(in_memory_sqlite):
     """Upgrade an empty database to the latest version"""
     schema = ModelSchema(in_memory_sqlite)
-    schema.upgrade(backup=False, set_views=False)
+    schema.upgrade(backup=False, set_views=False, upgrade_spatialite_version=False)
     assert schema.get_version() == get_schema_version()
     assert in_memory_sqlite.get_engine().has_table("v2_connection_nodes")
 
 
 def test_full_upgrade_with_preexisting_version(south_latest_sqlite):
     """Upgrade an empty database to the latest version"""
     schema = ModelSchema(south_latest_sqlite)
-    schema.upgrade(backup=False, set_views=False)
+    schema.upgrade(backup=False, set_views=False, upgrade_spatialite_version=False)
     assert schema.get_version() == get_schema_version()
     assert south_latest_sqlite.get_engine().has_table("v2_connection_nodes")
 
 
 def test_full_upgrade_oldest(oldest_sqlite):
     """Upgrade a legacy database to the latest version"""
     schema = ModelSchema(oldest_sqlite)
-    schema.upgrade(backup=False, set_views=False)
+    schema.upgrade(backup=False, set_views=False, upgrade_spatialite_version=False)
     assert schema.get_version() == get_schema_version()
     assert oldest_sqlite.get_engine().has_table("v2_connection_nodes")
 
@@ -141,7 +140,9 @@ def test_upgrade_south_not_latest_errors(in_memory_sqlite):
         schema, "get_version", return_value=constants.LATEST_SOUTH_MIGRATION_ID - 1
     ):
         with pytest.raises(errors.MigrationMissingError):
-            schema.upgrade(backup=False, set_views=False)
+            schema.upgrade(
+                backup=False, set_views=False, upgrade_spatialite_version=False
+            )
 
 
 def test_upgrade_with_backup(threedi_db):
@@ -153,7 +154,9 @@ def test_upgrade_with_backup(threedi_db):
         "threedi_modelchecker.schema._upgrade_database", side_effect=RuntimeError
     ) as upgrade, mock.patch.object(schema, "get_version", return_value=199):
         with pytest.raises(RuntimeError):
-            schema.upgrade(backup=True, set_views=False)
+            schema.upgrade(
+                backup=True, set_views=False, upgrade_spatialite_version=False
+            )
 
     (db,), kwargs = upgrade.call_args
     assert db is not threedi_db
@@ -166,20 +169,33 @@ def test_upgrade_without_backup(threedi_db):
         "threedi_modelchecker.schema._upgrade_database", side_effect=RuntimeError
     ) as upgrade, mock.patch.object(schema, "get_version", return_value=199):
         with pytest.raises(RuntimeError):
-            schema.upgrade(backup=False, set_views=False)
+            schema.upgrade(
+                backup=False, set_views=False, upgrade_spatialite_version=False
+            )
 
     (db,), kwargs = upgrade.call_args
     assert db is threedi_db
 
 
 def test_set_views(oldest_sqlite):
-    """Make sure that the views are regenerated
-    """
+    """Make sure that the views are regenerated"""
     schema = ModelSchema(oldest_sqlite)
-    schema.upgrade(backup=False, set_views=True)
+    schema.upgrade(backup=False, set_views=True, upgrade_spatialite_version=False)
     assert schema.get_version() == get_schema_version()
 
     # Test all views
     with oldest_sqlite.session_scope() as session:
         for view_name in ALL_VIEWS:
             session.execute(f"SELECT * FROM {view_name} LIMIT 1").fetchall()
+
+
+def test_upgrade_spatialite_3(oldest_sqlite):
+    lib_version, file_version_before = get_spatialite_version(oldest_sqlite)
+    if lib_version == file_version_before:
+        pytest.skip("Nothing to test: spatialite library version equals file version")
+
+    schema = ModelSchema(oldest_sqlite)
+    schema.upgrade(backup=False, upgrade_spatialite_version=True)
+
+    _, file_version_after = get_spatialite_version(oldest_sqlite)
+    assert file_version_after == 4

tests/test_spatalite_versions.py

@@ -0,0 +1,34 @@
+from geoalchemy2 import func as geo_func
+from threedi_modelchecker.spatialite_versions import get_spatialite_version, copy_model
+from threedi_modelchecker.threedi_model import models
+
+
+def test_get_spatialite_version(empty_sqlite_v3):
+    lib_version, file_version = get_spatialite_version(empty_sqlite_v3)
+    assert lib_version in (3, 4, 5)
+    assert file_version == 3
+
+
+def test_copy_model(empty_sqlite_v3, empty_sqlite_v3_clone):
+    obj = models.ConnectionNode(
+        id=3, code="test", the_geom="SRID=4326;POINT(-71.064544 42.287870)"
+    )
+    with empty_sqlite_v3.session_scope() as session:
+        session.add(obj)
+        session.commit()
+
+    copy_model(empty_sqlite_v3, empty_sqlite_v3_clone, models.ConnectionNode)
+
+    assert models.ConnectionNode.__table__.columns
+
+    with empty_sqlite_v3_clone.session_scope() as session:
+        records = list(
+            session.query(
+                models.ConnectionNode.id,
+                models.ConnectionNode.code,
+                geo_func.ST_AsText(models.ConnectionNode.the_geom),
+                models.ConnectionNode.the_geom_linestring,
+            )
+        )
+
+        assert records == [(3, "test", "POINT(-71.064544 42.28787)", None)]

threedi_modelchecker/migrations/env.py

@@ -1,5 +1,6 @@
 from alembic import context
 from sqlalchemy import create_engine
+from threedi_modelchecker.threedi_model import constants
 from threedi_modelchecker.threedi_model.models import Base
 
 import os
@@ -30,7 +31,11 @@ def run_migrations_online():
         connectable = create_engine(get_url())
 
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=constants.VERSION_TABLE_NAME,
+        )
 
         with context.begin_transaction():
             context.run_migrations()