Merge pull request #46 from OpenGeoscience/model-changes

Networks & File Items

sample_data/datasets.json

@@ -32,11 +32,6 @@
                 "path": "boston/commuter_rail.zip"
             }
         ],
-        "network_options": {
-            "connection_column": "LINE_BRNCH",
-            "connection_column_delimiter": "/",
-            "node_id_column": "STATION"
-        },
         "style_options": {
             "color_property": "gray",
             "outline": "black"

sample_data/ingest_sample_data.py

@@ -105,31 +105,19 @@ def ingest_datasets(include_large=False, dataset_indexes=None):
     with open('sample_data/datasets.json') as datasets_json:
         data = json.load(datasets_json)
         for index, dataset in enumerate(data):
-            # Grab fields specific to dataset classification
-            network_options = dataset.get('network_options')
-            region_options = dataset.get('region_options')
-
             if dataset_indexes is None or index in dataset_indexes:
                 print('\t- ', dataset['name'])
                 existing = Dataset.objects.filter(name=dataset['name'])
                 if existing.count():
                     dataset_for_conversion = existing.first()
                 else:
-                    # Determine classification
-                    classification = Dataset.Classification.OTHER
-                    if network_options:
-                        classification = Dataset.Classification.NETWORK
-                    elif region_options:
-                        classification = Dataset.Classification.REGION
-
                     # Create dataset
                     new_dataset = Dataset.objects.create(
                         name=dataset['name'],
                         description=dataset['description'],
                         category=dataset['category'],
                         dataset_type=dataset.get('type', 'vector').upper(),
                         metadata=dataset.get('metadata', {}),
-                        classification=classification,
                     )
                     print('\t', f'Dataset {new_dataset.name} created.')
                     for index, file_info in enumerate(dataset.get('files', [])):
@@ -145,8 +133,8 @@ def ingest_datasets(include_large=False, dataset_indexes=None):
                     print('\t', f'Converting data for {dataset_for_conversion.name}...')
                     dataset_for_conversion.spawn_conversion_task(
                         style_options=dataset.get('style_options'),
-                        network_options=network_options,
-                        region_options=region_options,
+                        network_options=dataset.get('network_options'),
+                        region_options=dataset.get('region_options'),
                         asynchronous=False,
                     )
                 else:

uvdat/core/admin.py

@@ -6,6 +6,7 @@
     Dataset,
     DerivedRegion,
     FileItem,
+    Network,
     NetworkEdge,
     NetworkNode,
     RasterMapLayer,
@@ -43,18 +44,24 @@ class RasterMapLayerAdmin(admin.ModelAdmin):
     list_display = ['id', 'get_dataset_name', 'index']
 
     def get_dataset_name(self, obj):
-        return obj.file_item.dataset.name
+        return obj.dataset.name
 
 
 class VectorMapLayerAdmin(admin.ModelAdmin):
     list_display = ['id', 'get_dataset_name', 'index']
 
     def get_dataset_name(self, obj):
-        return obj.file_item.dataset.name
+        return obj.dataset.name
 
 
 class VectorFeatureAdmin(admin.ModelAdmin):
-    list_display = ['id', 'map_layer']
+    list_display = ['id', 'get_dataset_name', 'get_map_layer_index']
+
+    def get_dataset_name(self, obj):
+        return obj.map_layer.dataset.name
+
+    def get_map_layer_index(self, obj):
+        return obj.map_layer.index
 
 
 class SourceRegionAdmin(admin.ModelAdmin):
@@ -74,18 +81,31 @@ def get_source_region_names(self, obj):
         return ', '.join(r.name for r in obj.source_regions.all())
 
 
-class NetworkEdgeAdmin(admin.ModelAdmin):
-    list_display = ['id', 'name', 'get_dataset_name']
+class NetworkAdmin(admin.ModelAdmin):
+    list_display = ['id', 'category', 'get_dataset_name']
 
     def get_dataset_name(self, obj):
         return obj.dataset.name
 
 
+class NetworkEdgeAdmin(admin.ModelAdmin):
+    list_display = ['id', 'name', 'get_network_id', 'get_dataset_name']
+
+    def get_network_id(self, obj):
+        return obj.network.id
+
+    def get_dataset_name(self, obj):
+        return obj.network.dataset.name
+
+
 class NetworkNodeAdmin(admin.ModelAdmin):
-    list_display = ['id', 'name', 'get_dataset_name', 'get_adjacent_node_names']
+    list_display = ['id', 'name', 'get_network_id', 'get_dataset_name', 'get_adjacent_node_names']
+
+    def get_network_id(self, obj):
+        return obj.network.id
 
     def get_dataset_name(self, obj):
-        return obj.dataset.name
+        return obj.network.dataset.name
 
     def get_adjacent_node_names(self, obj):
         return ', '.join(n.name for n in obj.get_adjacent_nodes())
@@ -104,6 +124,7 @@ class SimulationResultAdmin(admin.ModelAdmin):
 admin.site.register(VectorFeature, VectorFeatureAdmin)
 admin.site.register(SourceRegion, SourceRegionAdmin)
 admin.site.register(DerivedRegion, DerivedRegionAdmin)
+admin.site.register(Network, NetworkAdmin)
 admin.site.register(NetworkNode, NetworkNodeAdmin)
 admin.site.register(NetworkEdge, NetworkEdgeAdmin)
 admin.site.register(SimulationResult, SimulationResultAdmin)

uvdat/core/migrations/0004_files_and_networks.py

@@ -0,0 +1,91 @@
+# Generated by Django 4.1 on 2024-07-08 20:01
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0003_nonunique_network_names'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='dataset',
+            name='classification',
+        ),
+        migrations.RemoveField(
+            model_name='networkedge',
+            name='dataset',
+        ),
+        migrations.RemoveField(
+            model_name='networknode',
+            name='dataset',
+        ),
+        migrations.RemoveField(
+            model_name='rastermaplayer',
+            name='file_item',
+        ),
+        migrations.RemoveField(
+            model_name='vectormaplayer',
+            name='file_item',
+        ),
+        migrations.AddField(
+            model_name='rastermaplayer',
+            name='dataset',
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset'
+            ),
+        ),
+        migrations.AddField(
+            model_name='vectormaplayer',
+            name='dataset',
+            field=models.ForeignKey(
+                null=True, on_delete=django.db.models.deletion.CASCADE, to='core.dataset'
+            ),
+        ),
+        migrations.CreateModel(
+            name='Network',
+            fields=[
+                (
+                    'id',
+                    models.BigAutoField(
+                        auto_created=True, primary_key=True, serialize=False, verbose_name='ID'
+                    ),
+                ),
+                ('category', models.CharField(max_length=25)),
+                ('metadata', models.JSONField(blank=True, null=True)),
+                (
+                    'dataset',
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name='networks',
+                        to='core.dataset',
+                    ),
+                ),
+            ],
+        ),
+        migrations.AddField(
+            model_name='networkedge',
+            name='network',
+            field=models.ForeignKey(
+                default=None,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name='edges',
+                to='core.network',
+            ),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='networknode',
+            name='network',
+            field=models.ForeignKey(
+                default=None,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name='nodes',
+                to='core.network',
+            ),
+            preserve_default=False,
+        ),
+    ]

uvdat/core/models/__init__.py

@@ -3,7 +3,7 @@
 from .dataset import Dataset
 from .file_item import FileItem
 from .map_layers import RasterMapLayer, VectorFeature, VectorMapLayer
-from .networks import NetworkEdge, NetworkNode
+from .networks import Network, NetworkEdge, NetworkNode
 from .regions import DerivedRegion, SourceRegion
 from .simulations import SimulationResult
 
@@ -17,6 +17,7 @@
     VectorFeature,
     SourceRegion,
     DerivedRegion,
+    Network,
     NetworkEdge,
     NetworkNode,
     SimulationResult,

uvdat/core/models/dataset.py

@@ -6,11 +6,6 @@ class DatasetType(models.TextChoices):
         VECTOR = 'VECTOR', 'Vector'
         RASTER = 'RASTER', 'Raster'
 
-    class Classification(models.TextChoices):
-        NETWORK = 'Network'
-        REGION = 'Region'
-        OTHER = 'Other'
-
     name = models.CharField(max_length=255, unique=True)
     description = models.TextField(null=True, blank=True)
     category = models.CharField(max_length=25)
@@ -20,9 +15,6 @@ class Classification(models.TextChoices):
         max_length=max(len(choice[0]) for choice in DatasetType.choices),
         choices=DatasetType.choices,
     )
-    classification = models.CharField(
-        max_length=16, choices=Classification.choices, default=Classification.OTHER
-    )
 
     def is_in_context(self, context_id):
         from uvdat.core.models import Context
@@ -58,34 +50,13 @@ def get_regions(self):
 
         return SourceRegion.objects.filter(dataset=self)
 
-    def get_network(self):
-        from uvdat.core.models import NetworkEdge, NetworkNode
-
-        network = {
-            'nodes': NetworkNode.objects.filter(dataset=self),
-            'edges': NetworkEdge.objects.filter(dataset=self),
-        }
-        if len(network.get('nodes')) == 0 and len(network.get('edges')) == 0:
-            return None
-        return network
-
-    def get_network_graph(self):
-        from uvdat.core.tasks.networks import get_dataset_network_graph
-
-        return get_dataset_network_graph(self)
-
-    def get_network_gcc(self, exclude_nodes):
-        from uvdat.core.tasks.networks import get_dataset_network_gcc
-
-        return get_dataset_network_gcc(self, exclude_nodes)
-
     def get_map_layers(self):
         """Return a queryset of either RasterMapLayer, or VectorMapLayer."""
         from uvdat.core.models import RasterMapLayer, VectorMapLayer
 
         if self.dataset_type == self.DatasetType.RASTER:
-            return RasterMapLayer.objects.filter(file_item__dataset=self)
+            return RasterMapLayer.objects.filter(dataset=self)
         if self.dataset_type == self.DatasetType.VECTOR:
-            return VectorMapLayer.objects.filter(file_item__dataset=self)
+            return VectorMapLayer.objects.filter(dataset=self)
 
         raise NotImplementedError(f'Dataset Type {self.dataset_type}')

uvdat/core/models/map_layers.py

@@ -9,17 +9,17 @@
 import large_image
 from s3_file_field import S3FileField
 
-from .file_item import FileItem
+from .dataset import Dataset
 
 
 class AbstractMapLayer(TimeStampedModel):
-    file_item = models.ForeignKey(FileItem, on_delete=models.CASCADE, null=True)
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, null=True)
     metadata = models.JSONField(blank=True, null=True)
     default_style = models.JSONField(blank=True, null=True)
     index = models.IntegerField(null=True)
 
     def is_in_context(self, context_id):
-        return self.file_item.is_in_context(context_id)
+        return self.dataset.is_in_context(context_id)
 
     class Meta:
         abstract = True

uvdat/core/models/networks.py

@@ -1,18 +1,41 @@
 from django.contrib.gis.db import models as geo_models
 from django.db import models
+import networkx as nx
 
 from .dataset import Dataset
 
 
+class Network(models.Model):
+    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='networks')
+    category = models.CharField(max_length=25)
+    metadata = models.JSONField(blank=True, null=True)
+
+    def is_in_context(self, context_id):
+        return self.dataset.is_in_context(context_id)
+
+    def get_graph(self):
+        from uvdat.core.tasks.networks import get_network_graph
+
+        return get_network_graph(self)
+
+    def get_gcc(self, exclude_nodes):
+        graph = self.get_graph()
+        graph.remove_nodes_from(exclude_nodes)
+        if graph.number_of_nodes == 0 or nx.number_connected_components(graph) == 0:
+            return []
+        gcc = max(nx.connected_components(graph), key=len)
+        return list(gcc)
+
+
 class NetworkNode(models.Model):
     name = models.CharField(max_length=255)
-    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='network_nodes')
+    network = models.ForeignKey(Network, on_delete=models.CASCADE, related_name='nodes')
     metadata = models.JSONField(blank=True, null=True)
     capacity = models.IntegerField(null=True)
     location = geo_models.PointField()
 
     def is_in_context(self, context_id):
-        return self.dataset.is_in_context(context_id)
+        return self.network.is_in_context(context_id)
 
     def get_adjacent_nodes(self) -> models.QuerySet:
         entering_node_ids = (
@@ -32,7 +55,7 @@ def get_adjacent_nodes(self) -> models.QuerySet:
 
 class NetworkEdge(models.Model):
     name = models.CharField(max_length=255)
-    dataset = models.ForeignKey(Dataset, on_delete=models.CASCADE, related_name='network_edges')
+    network = models.ForeignKey(Network, on_delete=models.CASCADE, related_name='edges')
     metadata = models.JSONField(blank=True, null=True)
     capacity = models.IntegerField(null=True)
     line_geometry = geo_models.LineStringField()
@@ -41,4 +64,4 @@ class NetworkEdge(models.Model):
     to_node = models.ForeignKey(NetworkNode, related_name='+', on_delete=models.CASCADE)
 
     def is_in_context(self, context_id):
-        return self.dataset.is_in_context(context_id)
+        return self.network.is_in_context(context_id)