diff --git a/Classes/Command/UnduplicateCommand.php b/Classes/Command/UnduplicateCommand.php index bed7302..f85aaf1 100644 --- a/Classes/Command/UnduplicateCommand.php +++ b/Classes/Command/UnduplicateCommand.php @@ -64,6 +64,16 @@ class UnduplicateCommand extends Command */ private $dryRun = false; + /** + * @var String|bool + */ + private $force = false; + + /** + * @var bool + */ + private $keepOldest = false; + /** * @var SymfonyStyle */ @@ -85,7 +95,8 @@ public function __construct($name = null, ConnectionPool $connectionPool = null) */ public function configure() { - $this->setDescription('Finds duplicates in sys_file and unduplicates them'); + $this->setDescription('Finds duplicates in sys_file and unduplicates them. + By default it will use the newest (highest uid) record as master and delete the older records.'); $this->setHelp( 'currently fix references in ' . LF . '- sys_file_reference::link ' . LF . @@ -115,6 +126,20 @@ public function configure() 'Only use this storage', -1 ) + ->addOption( + 'force', + 'f', + InputOption::VALUE_OPTIONAL, + 'Force keep or overwrite of metadata of the master record in case of conflict. Possible values: keep, keep-nonempty. + Default: overwrite. Keep-nonempty is keeping only nonempty metadata in master, but updating the empty.', + false + ) + ->addOption( + 'keep-oldest', + 'o', + InputOption::VALUE_NONE, + 'Use the oldest record as master instead of the newest', + ) ->addOption( 'meta-fields', 'm', @@ -146,6 +171,11 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->updateReferenceIndex($input); $this->dryRun = $input->getOption('dry-run'); + $this->force = $input->getOption('force'); // force will be null if no value is passed -> overwrite + if ($this->force === null) { + $this->force = 'overwrite'; + } + $this->keepOldest = $input->getOption('keep-oldest'); $onlyThisIdentifier = $input->getOption('identifier'); $onlyThisStorage = (int)$input->getOption('storage'); @@ -258,7 +288,7 @@ private function findDuplicateFilesForIdentifier(string $identifier, int $storag 'storage', $fileQueryBuilder->createNamedParameter($storage, Connection::PARAM_INT) ) - )->orderBy('uid', 'DESC'); + )->orderBy('uid', $this->keepOldest ? 'ASC' : 'DESC'); $whereClause = 'MD5(identifier) = MD5(' . $fileQueryBuilder->createNamedParameter($identifier, \PDO::PARAM_STR) . ')'; $fileQueryBuilder->add('where', $whereClause); @@ -370,10 +400,11 @@ public function getSysRefIndexData(int $oldFileUid): Result public function updateMetadataRecord(int $masterFileUid, array $referenceRow): bool { - $metadata = $this->isMetadataRecordPopulated($referenceRow['ref_uid']); + $oldMetadata = $this->isMetadataRecordPopulated($referenceRow['ref_uid']); $masterFileMetadata = $this->isMetadataRecordPopulated($masterFileUid); + $masterEmoty = !$masterFileMetadata; - if (!$metadata || $metadata === $masterFileMetadata) { // check if record is empty or if the values are the same as in master + if (!$oldMetadata || $oldMetadata === $masterFileMetadata) { // check if record is empty or if the values are the same as in master $this->output->writeln('Deleting old metadata record'); if (!$this->dryRun) { @@ -381,11 +412,24 @@ public function updateMetadataRecord(int $masterFileUid, array $referenceRow): b $this->deleteReference($referenceRow); } - } elseif ($metadata && !$masterFileMetadata) { // check if master record has metadata, if not, copy the old ones - $this->output->writeln('Old metadata is not empty and master is empty, copying values to master'); + } elseif ($oldMetadata && ($masterEmoty || $this->force !== false)) { // check if master record has metadata, if not, copy the old ones + + if ($masterEmoty) { + $this->output->writeln('Old metadata is not empty and master is empty, copying values to master. Deleting old metadata record'); + } else if ($this->force !== false) { + if ($this->force !== 'keep') { + $this->output->writeln('Force overwriting metadata in master. Deleting old metadata record'); + } else { + $this->output->writeln('Force keeping metadata in master. Deleting old metadata record. Action may be required.'); + } + } if (!$this->dryRun) { - $this->updateMasterFileMetadata($masterFileUid, $metadata); + if ($this->force === false || + $this->force === 'overwrite' || + $masterEmoty && $this->force === 'keep-nonempty') { + $this->updateMasterFileMetadata($masterFileUid, $oldMetadata); + } $this->deleteReferencedRecord($referenceRow); $this->deleteReference($referenceRow); } diff --git a/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv new file mode 100644 index 0000000..1055086 --- /dev/null +++ b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv @@ -0,0 +1,18 @@ +"sys_file",,, +,"uid","identifier","storage" +,2,"/test/abc.jpg",1 +,4,"/test/abcd.jpg",1 +,6,"/test/abcde.jpg",1 +,8,"/test/abcdef.jpg",1 +"sys_refindex",,,,,,, +,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid" +,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2" +,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4" +,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6" +,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8" +"sys_file_metadata",,, +,"uid","file","description" +,2,2,"ABC" +,4,4,"ABC" +,6,6,"ABC" +,8,8,"" diff --git a/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv new file mode 100644 index 0000000..a07faaa --- /dev/null +++ b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv @@ -0,0 +1,18 @@ +"sys_file",,, +,"uid","identifier","storage" +,2,"/test/abc.jpg",1 +,4,"/test/abcd.jpg",1 +,6,"/test/abcde.jpg",1 +,8,"/test/abcdef.jpg",1 +"sys_refindex",,,,,,, +,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid" +,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2" +,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4" +,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6" +,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8" +"sys_file_metadata",,, +,"uid","file","description" +,2,2,"ABC" +,4,4,"ABC" +,6,6,"ABC" +,8,8,"ABC" diff --git a/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv new file mode 100644 index 0000000..30ea217 --- /dev/null +++ b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv @@ -0,0 +1,18 @@ +"sys_file",,, +,"uid","identifier","storage" +,2,"/test/abc.jpg",1 +,4,"/test/abcd.jpg",1 +,6,"/test/abcde.jpg",1 +,8,"/test/abcdef.jpg",1 +"sys_refindex",,,,,,, +,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid" +,"507cf53279411321324cd2f424e0682e","sys_file_metadata","2","","","sys_file","2" +,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4" +,"507cf53279411321324cd2f424e0686e","sys_file_metadata","6","","","sys_file","6" +,"507cf53279411321324cd2f424e0688e","sys_file_metadata","8","","","sys_file","8" +"sys_file_metadata",,, +,"uid","file","description" +,2,2,"ABC" +,4,4,"ABCD" +,6,6,"ABC" +,8,8,"ABC" diff --git a/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv new file mode 100644 index 0000000..78b5df3 --- /dev/null +++ b/Tests/Functional/Command/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv @@ -0,0 +1,21 @@ +"sys_file",,, +,"uid","identifier","storage" +,1,"/test/abc.jpg",1 +,3,"/test/abcd.jpg",1 +,4,"/test/abcd.jpg",1 +,5,"/test/abcde.jpg",1 +,7,"/test/abcdef.jpg",1 +"sys_refindex",,,,,,, +,"hash","tablename","recuid","field","softref_key","ref_table","ref_uid" +,"507cf53279411321324cd2f424e0681e","sys_file_metadata","1","","","sys_file","1" +,"507cf53279411321324cd2f424e0683e","sys_file_metadata","3","","","sys_file","3" +,"507cf53279411321324cd2f424e0684e","sys_file_metadata","4","","","sys_file","4" +,"507cf53279411321324cd2f424e0685e","sys_file_metadata","5","","","sys_file","5" +,"507cf53279411321324cd2f424e0687e","sys_file_metadata","7","","","sys_file","7" +"sys_file_metadata",,, +,"uid","file","description" +,1,1,"ABC" +,3,3,"ABCD" +,4,4,"ABC" +,5,5,"ABC" +,7,7,"ABC" diff --git a/Tests/Functional/Command/UnduplicateCommandTest.php b/Tests/Functional/Command/UnduplicateCommandTest.php index aa4a371..1061b5f 100644 --- a/Tests/Functional/Command/UnduplicateCommandTest.php +++ b/Tests/Functional/Command/UnduplicateCommandTest.php @@ -89,6 +89,50 @@ class UnduplicateCommandTest extends FunctionalTestCase self::assertEquals(0, $result['status']); } + #[Test] public function unduplicateCommandKeepOldestWithMetadata(): void + { + $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv'); + + $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --keep-oldest'); + + // the references are updated, so that the newer sys_file entry (uid=2) is used + $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_oldest_RESULT.csv'); + self::assertEquals(0, $result['status']); + } + + #[Test] public function unduplicateCommandForceOverwriteWithMetadata(): void + { + $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv'); + + $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force'); + + // the references are updated, so that the newer sys_file entry (uid=2) is used + $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_overwrite_RESULT.csv'); + self::assertEquals(0, $result['status']); + } + + #[Test] public function unduplicateCommandForceKeepWithMetadata(): void + { + $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv'); + + $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force keep'); + + // the references are updated, so that the newer sys_file entry (uid=2) is used + $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_keep_RESULT.csv'); + self::assertEquals(0, $result['status']); + } + + #[Test] public function unduplicateCommandForceKeepNonEmptyWithMetadata(): void + { + $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv'); + + $result = $this->executeConsoleCommand(self::BASE_COMMAND . ' --force keep-nonempty'); + + // the references are updated, so that the newer sys_file entry (uid=2) is used + $this->assertCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_force_keep_nonempty_RESULT.csv'); + self::assertEquals(0, $result['status']); + } + /** * Provide a processed file for the test run, so that it can be deleted * @var array