Skip to content

Commit

Permalink
Sk/csv str,error table mod (#3010)
Browse files Browse the repository at this point in the history
* Updated error table

* Force str data type

* Update error table
  • Loading branch information
gsa-suk authored Dec 15, 2023
1 parent 4c3b42e commit e8aa48e
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from django.core.management.base import BaseCommand
from django.conf import settings
from django.apps import apps
from collections import defaultdict

logger = logging.getLogger(__name__)
logger.setLevel(logging.WARNING)
Expand Down Expand Up @@ -125,10 +126,11 @@ def get_model_name(self, name):
return None

def load_data(self, file, model_obj, chunk_size):
dtypes = defaultdict(lambda: str)
print("Starting load data to postgres")
file.seek(0)
rows_loaded = 0
for df in pd.read_csv(file, iterator=True, chunksize=chunk_size):
for df in pd.read_csv(file, iterator=True, chunksize=chunk_size, dtype=dtypes):
# Each row is a dictionary. The columns are the
# correct names for our model. So, this should be a
# clean way to load the model from a row.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
# Generated by Django 4.2.6 on 2023-12-15 18:19

from django.db import migrations, models


class Migration(migrations.Migration):
dependencies = [
(
"census_historical_migration",
"0002_reportmigrationstatus_migrationerrordetail",
),
]

operations = [
migrations.RemoveField(
model_name="migrationerrordetail",
name="error_stack",
),
migrations.RemoveField(
model_name="migrationerrordetail",
name="error_summary",
),
migrations.AddField(
model_name="migrationerrordetail",
name="detail",
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name="migrationerrordetail",
name="exception_class",
field=models.TextField(blank=True, null=True),
),
migrations.AddField(
model_name="migrationerrordetail",
name="tag",
field=models.TextField(blank=True, null=True),
),
]
5 changes: 3 additions & 2 deletions backend/census_historical_migration/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -457,5 +457,6 @@ class MigrationErrorDetail(models.Model):
report_migration_status = models.ForeignKey(
ReportMigrationStatus, on_delete=models.CASCADE
)
error_summary = models.CharField(blank=True, null=True)
error_stack = models.TextField()
tag = models.TextField(blank=True, null=True)
exception_class = models.TextField(blank=True, null=True)
detail = models.TextField(blank=True, null=True)

0 comments on commit e8aa48e

Please sign in to comment.