Added new options to allow forced updates and to keep old records. See issue #16 and #17.

Classes/Command/UnduplicateCommand.php

@@ -64,6 +64,16 @@ class UnduplicateCommand extends Command
      */
     private $dryRun = false;
 
+    /**
+     * @var String|bool
+     */
+    private $force = false;
+
+    /**
+     * @var bool
+     */
+    private $keepOldest = false;
+
     /**
      * @var SymfonyStyle
      */
@@ -85,7 +95,8 @@ public function __construct($name = null, ConnectionPool $connectionPool = null)
      */
     public function configure()
     {
-        $this->setDescription('Finds duplicates in sys_file and unduplicates them');
+        $this->setDescription('Finds duplicates in sys_file and unduplicates them.
+        By default it will use the newest (highest uid) record as master and delete the older records.');
         $this->setHelp(
             'currently fix references in ' . LF .
             '- sys_file_reference::link ' . LF .
@@ -115,6 +126,20 @@ public function configure()
                 'Only use this storage',
                 -1
             )
+            ->addOption(
+                'force',
+                'f',
+                InputOption::VALUE_OPTIONAL,
+                'Force keep or overwrite of metadata of the master record in case of conflict. Possible values: keep, keep-nonempty.
+                Default: overwrite. Keep-nonempty is keeping only nonempty metadata in master, but updating the empty.',
+                false
+            )
+            ->addOption(
+                'keep-oldest',
+                'o',
+                InputOption::VALUE_NONE,
+                'Use the oldest record as master instead of the newest',
+            )
             ->addOption(
                 'meta-fields',
                 'm',
@@ -146,6 +171,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         $this->updateReferenceIndex($input);
 
         $this->dryRun = $input->getOption('dry-run');
+        $this->force = $input->getOption('force'); // force will be null if no value is passed -> overwrite
+        if ($this->force === null) {
+            $this->force = 'overwrite';
+        }
+        $this->keepOldest = $input->getOption('keep-oldest');
         $onlyThisIdentifier = $input->getOption('identifier');
         $onlyThisStorage = (int)$input->getOption('storage');
 
@@ -258,7 +288,7 @@ private function findDuplicateFilesForIdentifier(string $identifier, int $storag
                     'storage',
                     $fileQueryBuilder->createNamedParameter($storage, Connection::PARAM_INT)
                 )
-            )->orderBy('uid', 'DESC');
+            )->orderBy('uid', $this->keepOldest ? 'ASC' : 'DESC');
 
         $whereClause = 'MD5(identifier) = MD5(' . $fileQueryBuilder->createNamedParameter($identifier, \PDO::PARAM_STR) . ')';
         $fileQueryBuilder->add('where', $whereClause);
@@ -370,22 +400,36 @@ public function getSysRefIndexData(int $oldFileUid): Result
     public function updateMetadataRecord(int $masterFileUid, array $referenceRow): bool
     {
 
-        $metadata = $this->isMetadataRecordPopulated($referenceRow['ref_uid']);
+        $oldMetadata = $this->isMetadataRecordPopulated($referenceRow['ref_uid']);
         $masterFileMetadata = $this->isMetadataRecordPopulated($masterFileUid);
+        $masterEmoty = !$masterFileMetadata;
 
-        if (!$metadata || $metadata === $masterFileMetadata) { // check if record is empty or if the values are the same as in master
+        if (!$oldMetadata || $oldMetadata === $masterFileMetadata) { // check if record is empty or if the values are the same as in master
             $this->output->writeln('<info>Deleting old metadata record</info>');
 
             if (!$this->dryRun) {
                 $this->deleteReferencedRecord($referenceRow);
                 $this->deleteReference($referenceRow);
             }
 
-        } elseif ($metadata && !$masterFileMetadata) { // check if master record has metadata, if not, copy the old ones
-            $this->output->writeln('<info>Old metadata is not empty and master is empty, copying values to master</info>');
+        } elseif ($oldMetadata && ($masterEmoty || $this->force !== false)) { // check if master record has metadata, if not, copy the old ones
+
+            if ($masterEmoty) {
+                $this->output->writeln('<info>Old metadata is not empty and master is empty, copying values to master. Deleting old metadata record</info>');
+            } else if ($this->force !== false) {
+                if ($this->force !== 'keep') {
+                    $this->output->writeln('<info>Force overwriting metadata in master. Deleting old metadata record</info>');
+                } else {
+                    $this->output->writeln('<info>Force keeping metadata in master. Deleting old metadata record. Action may be required.</info>');
+                }
+            }
 
             if (!$this->dryRun) {
-                $this->updateMasterFileMetadata($masterFileUid, $metadata);
+                if ($this->force === false ||
+                    $this->force === 'overwrite' ||
+                    $masterEmoty && $this->force === 'keep-nonempty') {
+                    $this->updateMasterFileMetadata($masterFileUid, $oldMetadata);
+                }
                 $this->deleteReferencedRecord($referenceRow);
                 $this->deleteReference($referenceRow);
             }

Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv

@@ -0,0 +1,18 @@
+"sys_file",,,
+,"uid","identifier","storage"
+,2,"/test/abc.jpg",1
+,4,"/test/abcd.jpg",1
+,6,"/test/abcde.jpg",1
+,8,"/test/abcdef.jpg",1
+"sys_refindex",,,,,,,
+,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
+,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2"
+,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
+,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6"
+,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8"
+"sys_file_metadata",,,
+,"uid","file","description"
+,2,2,"ABC"
+,4,4,"ABC"
+,6,6,"ABC"
+,8,8,""

Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv

@@ -0,0 +1,18 @@
+"sys_file",,,
+,"uid","identifier","storage"
+,2,"/test/abc.jpg",1
+,4,"/test/abcd.jpg",1
+,6,"/test/abcde.jpg",1
+,8,"/test/abcdef.jpg",1
+"sys_refindex",,,,,,,
+,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
+,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2"
+,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
+,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6"
+,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8"
+"sys_file_metadata",,,
+,"uid","file","description"
+,2,2,"ABC"
+,4,4,"ABC"
+,6,6,"ABC"
+,8,8,"ABC"

Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv

@@ -0,0 +1,18 @@
+"sys_file",,,
+,"uid","identifier","storage"
+,2,"/test/abc.jpg",1
+,4,"/test/abcd.jpg",1
+,6,"/test/abcde.jpg",1
+,8,"/test/abcdef.jpg",1
+"sys_refindex",,,,,,,
+,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
+,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2"
+,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
+,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6"
+,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8"
+"sys_file_metadata",,,
+,"uid","file","description"
+,2,2,"ABC"
+,4,4,"ABCD"
+,6,6,"ABC"
+,8,8,"ABC"

Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv

@@ -0,0 +1,21 @@
+"sys_file",,,
+,"uid","identifier","storage"
+,1,"/test/abc.jpg",1
+,3,"/test/abcd.jpg",1
+,4,"/test/abcd.jpg",1
+,5,"/test/abcde.jpg",1
+,7,"/test/abcdef.jpg",1
+"sys_refindex",,,,,,,
+,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid"
+,"507cf53279411321324cd2f424e0681e","sys_file_metadata","1","","","sys_file","1"
+,"507cf53279411321324cd2f424e0683e","sys_file_metadata","3","","","sys_file","3"
+,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4"
+,"507cf53279411321324cd2f424e0685e","sys_file_metadata","5","","","sys_file","5"
+,"507cf53279411321324cd2f424e0687e","sys_file_metadata","7","","","sys_file","7"
+"sys_file_metadata",,,
+,"uid","file","description"
+,1,1,"ABC"
+,3,3,"ABCD"
+,4,4,"ABC"
+,5,5,"ABC"
+,7,7,"ABC"

Tests/Functional/Command/UnduplicateCommandTest.php

@@ -89,6 +89,50 @@ class UnduplicateCommandTest extends FunctionalTestCase
         self::assertEquals(0, $result['status']);
     }
 
+    #[Test] public function unduplicateCommandKeepOldestWithMetadata(): void
+    {
+        $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
+
+        $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --keep-oldest');
+
+        // the references are updated, so that the newer sys_file entry (uid=2) is used
+        $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv');
+        self::assertEquals(0, $result['status']);
+    }
+
+    #[Test] public function unduplicateCommandForceOverwriteWithMetadata(): void
+    {
+        $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
+
+        $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force');
+
+        // the references are updated, so that the newer sys_file entry (uid=2) is used
+        $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv');
+        self::assertEquals(0, $result['status']);
+    }
+
+    #[Test] public function unduplicateCommandForceKeepWithMetadata(): void
+    {
+        $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
+
+        $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force keep');
+
+        // the references are updated, so that the newer sys_file entry (uid=2) is used
+        $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv');
+        self::assertEquals(0, $result['status']);
+    }
+
+    #[Test] public function unduplicateCommandForceKeepNonEmptyWithMetadata(): void
+    {
+        $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv');
+
+        $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force keep-nonempty');
+
+        // the references are updated, so that the newer sys_file entry (uid=2) is used
+        $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv');
+        self::assertEquals(0, $result['status']);
+    }
+
     /**
      * Provide a processed file for the test run, so that it can be deleted
      * @var array<string, string>