feat: schema diff sql tools (#17116)

Author: jrasm91

.github/workflows/test.yml

@@ -525,7 +525,7 @@ jobs:
 
       - name: Generate new migrations
         continue-on-error: true
-        run: npm run typeorm:migrations:generate ./src/migrations/TestMigration
+        run: npm run migrations:generate TestMigration
 
       - name: Find file changes
         uses: tj-actions/verify-changed-files@a1c6acee9df209257a246f2cc6ae8cb6581c1edf # v20
@@ -538,7 +538,7 @@ jobs:
         run: |
           echo "ERROR: Generated migration files not up to date!"
           echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}"
-          cat ./src/migrations/*-TestMigration.ts
+          cat ./src/*-TestMigration.ts
           exit 1
 
       - name: Run SQL generation

server/eslint.config.mjs

@@ -77,6 +77,14 @@ export default [
           ],
         },
       ],
+
+      '@typescript-eslint/no-unused-vars': [
+        'warn',
+        {
+          argsIgnorePattern: '^_',
+          varsIgnorePattern: '^_',
+        },
+      ],
     },
   },
 ];

server/package.json

@@ -23,8 +23,8 @@
     "test:medium": "vitest --config test/vitest.config.medium.mjs",
     "typeorm": "typeorm",
     "lifecycle": "node ./dist/utils/lifecycle.js",
-    "typeorm:migrations:create": "typeorm migration:create",
-    "typeorm:migrations:generate": "typeorm migration:generate -d ./dist/bin/database.js",
+    "migrations:generate": "node ./dist/bin/migrations.js generate",
+    "migrations:create": "node ./dist/bin/migrations.js create",
     "typeorm:migrations:run": "typeorm migration:run -d ./dist/bin/database.js",
     "typeorm:migrations:revert": "typeorm migration:revert -d ./dist/bin/database.js",
     "typeorm:schema:drop": "typeorm query -d ./dist/bin/database.js 'DROP schema public cascade; CREATE schema public;'",

server/src/bin/migrations.ts

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+process.env.DB_URL = 'postgres://postgres:postgres@localhost:5432/immich';
+
+import { writeFileSync } from 'node:fs';
+import postgres from 'postgres';
+import { ConfigRepository } from 'src/repositories/config.repository';
+import { DatabaseTable, schemaDiff, schemaFromDatabase, schemaFromDecorators } from 'src/sql-tools';
+import 'src/tables';
+
+const main = async () => {
+  const command = process.argv[2];
+  const name = process.argv[3] || 'Migration';
+
+  switch (command) {
+    case 'debug': {
+      await debug();
+      return;
+    }
+
+    case 'create': {
+      create(name, [], []);
+      return;
+    }
+
+    case 'generate': {
+      await generate(name);
+      return;
+    }
+
+    default: {
+      console.log(`Usage:
+  node dist/bin/migrations.js create <name>
+  node dist/bin/migrations.js generate <name>
+`);
+    }
+  }
+};
+
+const debug = async () => {
+  const { up, down } = await compare();
+  const upSql = '-- UP\n' + up.asSql({ comments: true }).join('\n');
+  const downSql = '-- DOWN\n' + down.asSql({ comments: true }).join('\n');
+  writeFileSync('./migrations.sql', upSql + '\n\n' + downSql);
+  console.log('Wrote migrations.sql');
+};
+
+const generate = async (name: string) => {
+  const { up, down } = await compare();
+  if (up.items.length === 0) {
+    console.log('No changes detected');
+    return;
+  }
+  create(name, up.asSql(), down.asSql());
+};
+
+const create = (name: string, up: string[], down: string[]) => {
+  const { filename, code } = asMigration(name, up, down);
+  const fullPath = `./src/${filename}`;
+  writeFileSync(fullPath, code);
+  console.log(`Wrote ${fullPath}`);
+};
+
+const compare = async () => {
+  const configRepository = new ConfigRepository();
+  const { database } = configRepository.getEnv();
+  const db = postgres(database.config.kysely);
+
+  const source = schemaFromDecorators();
+  const target = await schemaFromDatabase(db, {});
+
+  console.log(source.warnings.join('\n'));
+
+  const isIncluded = (table: DatabaseTable) => source.tables.some(({ name }) => table.name === name);
+  target.tables = target.tables.filter((table) => isIncluded(table));
+
+  const up = schemaDiff(source, target, { ignoreExtraTables: true });
+  const down = schemaDiff(target, source, { ignoreExtraTables: false });
+
+  return { up, down };
+};
+
+const asMigration = (name: string, up: string[], down: string[]) => {
+  const timestamp = Date.now();
+
+  const upSql = up.map((sql) => `    await queryRunner.query(\`${sql}\`);`).join('\n');
+  const downSql = down.map((sql) => `    await queryRunner.query(\`${sql}\`);`).join('\n');
+  return {
+    filename: `${timestamp}-${name}.ts`,
+    code: `import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class ${name}${timestamp} implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+${upSql}
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+${downSql}
+  }
+}
+`,
+  };
+};
+
+main()
+  .then(() => {
+    process.exit(0);
+  })
+  .catch((error) => {
+    console.error(error);
+    console.log('Something went wrong');
+    process.exit(1);
+  });

server/src/db.d.ts

@@ -4,8 +4,9 @@
  */
 
 import type { ColumnType } from 'kysely';
-import { OnThisDayData } from 'src/entities/memory.entity';
 import { AssetType, MemoryType, Permission, SyncEntityType } from 'src/enum';
+import { UserTable } from 'src/tables/user.table';
+import { OnThisDayData } from 'src/types';
 
 export type ArrayType<T> = ArrayTypeImpl<T> extends (infer U)[] ? U[] : ArrayTypeImpl<T>;
 
@@ -410,26 +411,6 @@ export interface UserMetadata {
   value: Json;
 }
 
-export interface Users {
-  createdAt: Generated<Timestamp>;
-  deletedAt: Timestamp | null;
-  email: string;
-  id: Generated<string>;
-  isAdmin: Generated<boolean>;
-  name: Generated<string>;
-  oauthId: Generated<string>;
-  password: Generated<string>;
-  profileChangedAt: Generated<Timestamp>;
-  profileImagePath: Generated<string>;
-  quotaSizeInBytes: Int8 | null;
-  quotaUsageInBytes: Generated<Int8>;
-  shouldChangePassword: Generated<boolean>;
-  status: Generated<string>;
-  storageLabel: string | null;
-  updatedAt: Generated<Timestamp>;
-  updateId: Generated<string>;
-}
-
 export interface UsersAudit {
   id: Generated<string>;
   userId: string;
@@ -495,7 +476,7 @@ export interface DB {
   tags_closure: TagsClosure;
   typeorm_metadata: TypeormMetadata;
   user_metadata: UserMetadata;
-  users: Users;
+  users: UserTable;
   users_audit: UsersAudit;
   'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat;
   version_history: VersionHistory;

server/src/entities/activity.entity.ts

@@ -1,27 +1,11 @@
 import { AlbumEntity } from 'src/entities/album.entity';
 import { UserEntity } from 'src/entities/user.entity';
 import { AlbumUserRole } from 'src/enum';
-import { Column, Entity, Index, JoinColumn, ManyToOne, PrimaryColumn } from 'typeorm';
 
-@Entity('albums_shared_users_users')
-// Pre-existing indices from original album <--> user ManyToMany mapping
-@Index('IDX_427c350ad49bd3935a50baab73', ['album'])
-@Index('IDX_f48513bf9bccefd6ff3ad30bd0', ['user'])
 export class AlbumUserEntity {
-  @PrimaryColumn({ type: 'uuid', name: 'albumsId' })
   albumId!: string;
-
-  @PrimaryColumn({ type: 'uuid', name: 'usersId' })
   userId!: string;
-
-  @JoinColumn({ name: 'albumsId' })
-  @ManyToOne(() => AlbumEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false })
   album!: AlbumEntity;
-
-  @JoinColumn({ name: 'usersId' })
-  @ManyToOne(() => UserEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false })
   user!: UserEntity;
-
-  @Column({ type: 'varchar', default: AlbumUserRole.EDITOR })
   role!: AlbumUserRole;
 }

server/src/entities/album-user.entity.ts

@@ -3,69 +3,22 @@ import { AssetEntity } from 'src/entities/asset.entity';
 import { SharedLinkEntity } from 'src/entities/shared-link.entity';
 import { UserEntity } from 'src/entities/user.entity';
 import { AssetOrder } from 'src/enum';
-import {
-  Column,
-  CreateDateColumn,
-  DeleteDateColumn,
-  Entity,
-  Index,
-  JoinTable,
-  ManyToMany,
-  ManyToOne,
-  OneToMany,
-  PrimaryGeneratedColumn,
-  UpdateDateColumn,
-} from 'typeorm';
 
-@Entity('albums')
 export class AlbumEntity {
-  @PrimaryGeneratedColumn('uuid')
   id!: string;
-
-  @ManyToOne(() => UserEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false })
   owner!: UserEntity;
-
-  @Column()
   ownerId!: string;
-
-  @Column({ default: 'Untitled Album' })
   albumName!: string;
-
-  @Column({ type: 'text', default: '' })
   description!: string;
-
-  @CreateDateColumn({ type: 'timestamptz' })
   createdAt!: Date;
-
-  @UpdateDateColumn({ type: 'timestamptz' })
   updatedAt!: Date;
-
-  @Index('IDX_albums_update_id')
-  @Column({ type: 'uuid', nullable: false, default: () => 'immich_uuid_v7()' })
   updateId?: string;
-
-  @DeleteDateColumn({ type: 'timestamptz' })
   deletedAt!: Date | null;
-
-  @ManyToOne(() => AssetEntity, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' })
   albumThumbnailAsset!: AssetEntity | null;
-
-  @Column({ comment: 'Asset ID to be used as thumbnail', nullable: true })
   albumThumbnailAssetId!: string | null;
-
-  @OneToMany(() => AlbumUserEntity, ({ album }) => album, { cascade: true, onDelete: 'CASCADE' })
   albumUsers!: AlbumUserEntity[];
-
-  @ManyToMany(() => AssetEntity, (asset) => asset.albums)
-  @JoinTable({ synchronize: false })
   assets!: AssetEntity[];
-
-  @OneToMany(() => SharedLinkEntity, (link) => link.album)
   sharedLinks!: SharedLinkEntity[];
-
-  @Column({ default: true })
   isActivityEnabled!: boolean;
-
-  @Column({ type: 'varchar', default: AssetOrder.DESC })
   order!: AssetOrder;
 }