Refactor and enhancements to saving of ML pipeline results

ami/jobs/migrations/0013_add_job_logs.py

@@ -0,0 +1,53 @@
+# Generated by Django 4.2.10 on 2024-12-17 20:01
+
+import django_pydantic_field.fields
+from django.db import migrations
+
+import ami.jobs.models
+
+
+def migrate_logs_forward(apps, schema_editor):
+    """Move logs from Job.progress to Job.logs"""
+    Job = apps.get_model("jobs", "Job")
+    jobs_to_update = []
+    for job in Job.objects.filter(progress__isnull=False):
+        if job.progress.logs or job.progress.errors:
+            # Move logs from progress to the new logs field
+            job.logs.stdout = job.progress.logs
+            job.logs.stderr = job.progress.errors
+            jobs_to_update.append(job)
+    # Update all jobs in a single query
+    Job.objects.bulk_update(jobs_to_update, ["logs"])
+
+
+def migrate_logs_backward(apps, schema_editor):
+    """Move logs from Job.logs back to Job.progress"""
+    Job = apps.get_model("jobs", "Job")
+    jobs_to_update = []
+    for job in Job.objects.filter(logs__isnull=False):
+        # Move logs back to progress
+        job.progress.logs = job.logs.stdout
+        job.progress.errors = job.logs.stderr
+        jobs_to_update.append(job)
+    # Update all jobs in a single query
+    Job.objects.bulk_update(jobs_to_update, ["progress"])
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("jobs", "0012_alter_job_limit"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="job",
+            name="logs",
+            field=django_pydantic_field.fields.PydanticSchemaField(
+                config=None, default={"stderr": [], "stdout": []}, schema=ami.jobs.models.JobLogs
+            ),
+        ),
+        migrations.RunPython(
+            migrate_logs_forward,
+            migrate_logs_backward,
+        ),
+    ]

ami/jobs/migrations/0014_alter_job_progress.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.10 on 2024-12-17 20:13
+
+import ami.jobs.models
+from django.db import migrations
+import django_pydantic_field.fields
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("jobs", "0013_add_job_logs"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="job",
+            name="progress",
+            field=django_pydantic_field.fields.PydanticSchemaField(
+                config=None,
+                default={"errors": [], "logs": [], "stages": [], "summary": {"progress": 0.0, "status": "CREATED"}},
+                schema=ami.jobs.models.JobProgress,
+            ),
+        ),
+    ]