fix: migrations to make postgresql compatible.

common/djangoapps/split_modulestore_django/migrations/0001_initial.py

@@ -1,15 +1,40 @@
 # Generated by Django 2.2.20 on 2021-05-07 18:29, manually modified to make "course_id" column case sensitive
 
 from django.conf import settings
-from django.db import migrations, models
+from django.db import migrations, models, connection
 import django.db.models.deletion
 import opaque_keys.edx.django.models
 import simple_history.models
 
 
+def generate_split_module_sql(db_engine):
+    if 'mysql' in db_engine:
+        return 'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;'
+    elif 'postgresql' in db_engine:
+        return """
+            ALTER TABLE split_modulestore_django_splitmodulestorecourseindex
+            ALTER COLUMN course_id TYPE VARCHAR(255),
+            ALTER COLUMN course_id SET NOT NULL;
+
+            ALTER TABLE split_modulestore_django_splitmodulestorecourseindex
+            ADD CONSTRAINT course_id_unique UNIQUE (course_id);
+        """
+
+
+def generate_split_history_module_sql(db_engine):
+    if 'mysql' in db_engine:
+        return 'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;'
+    elif 'postgresql' in db_engine:
+        return """
+            ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex
+            ALTER COLUMN course_id TYPE VARCHAR(255),
+            ALTER COLUMN course_id SET NOT NULL,
+            ALTER COLUMN course_id SET DATA TYPE VARCHAR(255) COLLATE "C";
+        """
 class Migration(migrations.Migration):
 
     initial = True
+    db_engine = connection.settings_dict['ENGINE']
 
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
@@ -65,11 +90,11 @@ class Migration(migrations.Migration):
         # Custom code: Convert columns to utf8_bin because we want to allow
         # case-sensitive comparisons for CourseKeys, which were case-sensitive in MongoDB
         migrations.RunSQL(
-            'ALTER TABLE split_modulestore_django_splitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL UNIQUE;',
+            generate_split_module_sql(db_engine),
             reverse_sql=migrations.RunSQL.noop,
         ),
         migrations.RunSQL(
-            'ALTER TABLE split_modulestore_django_historicalsplitmodulestorecourseindex MODIFY course_id varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL;',
+            generate_split_history_module_sql(db_engine),
             reverse_sql=migrations.RunSQL.noop,
         ),
     ]

lms/djangoapps/commerce/migrations/0001_data__add_ecommerce_service_user.py

@@ -1,22 +1,24 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.contrib.auth.models import User
-from django.db import migrations, models
+from django.db import migrations, models, transaction
 
 USERNAME = settings.ECOMMERCE_SERVICE_WORKER_USERNAME
 EMAIL = USERNAME + '@fake.email'
 
 def forwards(apps, schema_editor):
     """Add the service user."""
     User = get_user_model()
-    user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL)
-    if created:
-        user.set_unusable_password()
-        user.save()
+    with transaction.atomic():
+        user, created = User.objects.get_or_create(username=USERNAME, email=EMAIL)
+        if created:
+            user.set_unusable_password()
+            user.save()
 
 def backwards(apps, schema_editor):
     """Remove the service user."""
-    User.objects.get(username=USERNAME, email=EMAIL).delete()
+    with transaction.atomic():
+        User.objects.get(username=USERNAME, email=EMAIL).delete()
 
 class Migration(migrations.Migration):
 

lms/djangoapps/courseware/fields.py

@@ -18,7 +18,7 @@ def db_type(self, connection):
             # is an alias for that (https://www.sqlite.org/autoinc.html). An unsigned integer
             # isn't an alias for ROWID, so we have to give up on the unsigned part.
             return "integer"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
+        elif "postgresql" in connection.settings_dict['ENGINE']:
             # Pg's bigserial is implicitly unsigned (doesn't allow negative numbers) and
             # goes 1-9.2x10^18
             return "BIGSERIAL"
@@ -30,7 +30,7 @@ def rel_db_type(self, connection):
             return "bigint UNSIGNED"
         elif connection.settings_dict['ENGINE'] == 'django.db.backends.sqlite3':
             return "integer"
-        elif connection.settings_dict['ENGINE'] == 'django.db.backends.postgresql_psycopg2':
+        elif "postgresql" in connection.settings_dict['ENGINE']:
             return "BIGSERIAL"
         else:
             return None

lms/djangoapps/courseware/migrations/0008_move_idde_to_edx_when.py

@@ -33,5 +33,5 @@ class Migration(migrations.Migration):
     ]
 
     operations = [
-        migrations.RunPython(move_overrides_to_edx_when)
+        migrations.RunPython(move_overrides_to_edx_when, reverse_code=migrations.RunPython.noop)
     ]

lms/djangoapps/courseware/migrations/0011_csm_id_bigint.py

@@ -1,28 +1,38 @@
 # Generated by Django 1.11.23 on 2019-08-28 15:50
 
-
 import lms.djangoapps.courseware.fields
 
 from django.conf import settings
-from django.db import migrations
+from django.db import migrations, models
 from django.db.migrations import AlterField
 
 class CsmBigInt(AlterField):
     '''
     Subclass AlterField migration class to split SQL between two different databases
-    We can't use the normal AlterField migration operation because Django generate and routes migrations at the model
+    We can't use the normal AlterField migration operation because Django generates and routes migrations at the model
     level and the coursewarehistoryextended_studentmodulehistoryextended table is in a different database
     '''
     def database_forwards(self, app_label, schema_editor, from_state, to_state):
         if hasattr(schema_editor.connection, 'is_in_memory_db') and schema_editor.connection.is_in_memory_db():
             # sqlite3 doesn't support 'MODIFY', so skipping during tests
             return
+
         to_model = to_state.apps.get_model(app_label, self.model_name)
+
         if schema_editor.connection.alias == 'student_module_history':
             if settings.FEATURES["ENABLE_CSMH_EXTENDED"]:
-              schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;")
+                if schema_editor.connection.vendor == 'mysql':
+                    schema_editor.execute("ALTER TABLE `coursewarehistoryextended_studentmodulehistoryextended` MODIFY `student_module_id` bigint UNSIGNED NOT NULL;")
+                elif schema_editor.connection.vendor == 'postgresql':
+                    schema_editor.execute("ALTER TABLE coursewarehistoryextended_studentmodulehistoryextended ALTER COLUMN student_module_id TYPE bigint;")
         elif self.allow_migrate_model(schema_editor.connection.alias, to_model):
-            schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;")
+            if schema_editor.connection.vendor == 'postgresql':
+                # For PostgreSQL
+                schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET DATA TYPE bigint;")
+                schema_editor.execute("ALTER TABLE courseware_studentmodule ALTER COLUMN id SET NOT NULL;")
+            else:
+                # For MySQL
+                schema_editor.execute("ALTER TABLE `courseware_studentmodule` MODIFY `id` bigint UNSIGNED AUTO_INCREMENT NOT NULL;")
 
     def database_backwards(self, app_label, schema_editor, from_state, to_state):
         # Make backwards migration a no-op, app will still work if column is wider than expected
@@ -33,6 +43,7 @@ class Migration(migrations.Migration):
     dependencies = [
         ('courseware', '0010_auto_20190709_1559'),
     ]
+
     if settings.FEATURES["ENABLE_CSMH_EXTENDED"]:
         dependencies.append(('coursewarehistoryextended', '0002_force_studentmodule_index'))
 

lms/djangoapps/coursewarehistoryextended/migrations/0001_initial.py

@@ -2,7 +2,6 @@
 
 from django.db import migrations, models
 import django.db.models.deletion
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
 from django.conf import settings
 
 def bump_pk_start(apps, schema_editor):
@@ -45,7 +44,7 @@ class Migration(migrations.Migration):
                 ('state', models.TextField(null=True, blank=True)),
                 ('grade', models.FloatField(null=True, blank=True)),
                 ('max_grade', models.FloatField(null=True, blank=True)),
-                ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
                 ('student_module', models.ForeignKey(to='courseware.StudentModule', on_delete=django.db.models.deletion.DO_NOTHING, db_constraint=False)),
             ],
             options={

lms/djangoapps/coursewarehistoryextended/models.py

@@ -19,7 +19,6 @@
 from django.dispatch import receiver
 
 from lms.djangoapps.courseware.models import BaseStudentModuleHistory, StudentModule
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
 
 
 class StudentModuleHistoryExtended(BaseStudentModuleHistory):
@@ -40,7 +39,7 @@ class Meta:
             ),
         ]
 
-    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+    id = models.BigAutoField(primary_key=True, serialize=False)  # pylint: disable=invalid-name
 
     student_module = models.ForeignKey(StudentModule, db_index=False, db_constraint=False, on_delete=models.DO_NOTHING)
 

lms/djangoapps/grades/migrations/0001_initial.py

@@ -3,8 +3,6 @@
 from django.db import migrations, models
 from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField
 
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
-
 
 class Migration(migrations.Migration):
 
@@ -17,7 +15,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
                 ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
-                ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
                 ('user_id', models.IntegerField()),
                 ('course_id', CourseKeyField(max_length=255)),
                 ('usage_key', UsageKeyField(max_length=255)),

lms/djangoapps/grades/migrations/0006_persistent_course_grades.py

@@ -3,8 +3,6 @@
 from django.db import migrations, models
 from opaque_keys.edx.django.models import CourseKeyField
 
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
-
 
 class Migration(migrations.Migration):
 
@@ -18,7 +16,7 @@ class Migration(migrations.Migration):
             fields=[
                 ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, verbose_name='created', editable=False)),
                 ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, verbose_name='modified', editable=False)),
-                ('id', UnsignedBigIntAutoField(serialize=False, primary_key=True)),
+                ('id', models.BigAutoField(primary_key=True, serialize=False)),
                 ('user_id', models.IntegerField(db_index=True)),
                 ('course_id', CourseKeyField(max_length=255)),
                 ('course_edited_timestamp', models.DateTimeField(verbose_name='Last content edit timestamp')),

lms/djangoapps/grades/migrations/0015_historicalpersistentsubsectiongradeoverride.py

@@ -1,14 +1,13 @@
 # Generated by Django 1.11.20 on 2019-06-05 13:59
 
 
-from django.conf import settings
-from django.db import migrations, models
 import django.db.models.deletion
 import simple_history.models
+from django.conf import settings
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):
-
     dependencies = [
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
         ('grades', '0014_persistentsubsectiongradeoverridehistory'),
@@ -28,15 +27,24 @@ class Migration(migrations.Migration):
                 ('history_id', models.AutoField(primary_key=True, serialize=False)),
                 ('history_date', models.DateTimeField()),
                 ('history_change_reason', models.CharField(max_length=100, null=True)),
-                ('history_type', models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
-                ('grade', models.ForeignKey(blank=True, db_constraint=False, null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='+', to='grades.PersistentSubsectionGrade')),
-                ('history_user', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to=settings.AUTH_USER_MODEL)),
+                ('history_type',
+                 models.CharField(choices=[('+', 'Created'), ('~', 'Changed'), ('-', 'Deleted')], max_length=1)),
+                ('history_user',
+                 models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+',
+                                   to=settings.AUTH_USER_MODEL)),
             ],
-            options={
-                'ordering': ('-history_date', '-history_id'),
-                'get_latest_by': 'history_date',
-                'verbose_name': 'historical persistent subsection grade override',
-            },
-            bases=(simple_history.models.HistoricalChanges, models.Model),
+        options = {
+        'ordering': ('-history_date', '-history_id'),
+        'get_latest_by': 'history_date',
+        'verbose_name': 'historical persistent subsection grade override',
+    },
+    bases = (simple_history.models.HistoricalChanges, models.Model),
+    ),
+        migrations.AddField(
+            model_name='historicalpersistentsubsectiongradeoverride',
+            name='grade',
+            field=models.ForeignKey(blank=True, db_constraint=False, null=True,
+                                    on_delete=django.db.models.deletion.DO_NOTHING, related_name='+',
+                                    to='grades.PersistentSubsectionGrade'),
         ),
     ]

lms/djangoapps/grades/models.py

@@ -26,7 +26,6 @@
 from opaque_keys.edx.keys import CourseKey, UsageKey
 from simple_history.models import HistoricalRecords
 
-from lms.djangoapps.courseware.fields import UnsignedBigIntAutoField
 from lms.djangoapps.grades import events  # lint-amnesty, pylint: disable=unused-import
 from openedx.core.lib.cache_utils import get_cache
 from lms.djangoapps.grades.signals.signals import (
@@ -327,7 +326,7 @@ class Meta:
         ]
 
     # primary key will need to be large for this table
-    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+    id = models.BigAutoField(primary_key=True)  # pylint: disable=invalid-name
 
     user_id = models.IntegerField(blank=False)
     course_id = CourseKeyField(blank=False, max_length=255)
@@ -581,7 +580,7 @@ class Meta:
         ]
 
     # primary key will need to be large for this table
-    id = UnsignedBigIntAutoField(primary_key=True)  # pylint: disable=invalid-name
+    id = models.BigAutoField(primary_key=True)  # pylint: disable=invalid-name
     user_id = models.IntegerField(blank=False, db_index=True)
     course_id = CourseKeyField(blank=False, max_length=255)
 

openedx/core/djangoapps/common_initialization/apps.py

@@ -2,8 +2,8 @@
 Common initialization app for the LMS and CMS
 """
 
-
 from django.apps import AppConfig
+from django.db import connection
 
 
 class CommonInitializationConfig(AppConfig):  # lint-amnesty, pylint: disable=missing-class-docstring
@@ -14,6 +14,7 @@ def ready(self):
         # Common settings validations for the LMS and CMS.
         from . import checks  # lint-amnesty, pylint: disable=unused-import
         self._add_mimetypes()
+        self._add_required_adapters()
 
     @staticmethod
     def _add_mimetypes():
@@ -26,3 +27,21 @@ def _add_mimetypes():
         mimetypes.add_type('application/x-font-opentype', '.otf')
         mimetypes.add_type('application/x-font-ttf', '.ttf')
         mimetypes.add_type('application/font-woff', '.woff')
+
+    @staticmethod
+    def _add_required_adapters():
+        """
+        Register CourseLocator in psycopg2 extensions
+        :return:
+        """
+        if 'postgresql' in connection.vendor.lower():
+            from opaque_keys.edx.locator import CourseLocator, LibraryLocator, BlockUsageLocator
+            from psycopg2.extensions import QuotedString, register_adapter
+
+            def adapt_course_locator(course_locator):
+                return QuotedString(course_locator._to_string())  # lint-amnesty, pylint: disable=protected-access
+
+            # Register the adapter
+            register_adapter(CourseLocator, adapt_course_locator)
+            register_adapter(LibraryLocator, adapt_course_locator)
+            register_adapter(BlockUsageLocator, adapt_course_locator)