From 0401c5dd608b2502a935f9b20fd67416b1d6f530 Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Thu, 18 Dec 2025 14:02:18 +0100 Subject: [PATCH 1/8] Update dependencies in composer.json for TYPO3 v13 --- composer.json | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index e1dbe73..e7041b9 100644 --- a/composer.json +++ b/composer.json @@ -4,9 +4,9 @@ "description": "Finds duplicates in sys_file and unduplicates them", "license": "GPL-2.0-or-later", "require": { - "php": "^8.1 || ^8.2 || ^8.3", + "php": "^8.2 || ^8.3 || ^8.4 || ^8.5", "ext-pdo": "*", - "typo3/cms-core": "^12.4" + "typo3/cms-core": "^13.4" }, "extra": { "typo3/cms": { @@ -34,8 +34,7 @@ } }, "require-dev": { - "netresearch/rte-ckeditor-image": "^12.0", - "phpunit/phpunit": "^11", - "typo3/testing-framework": "8.2.0" + "netresearch/rte-ckeditor-image": "^13.0", + "typo3/testing-framework": "^9.3" } } From fe2766d8b1ec66e30c5b79e4480a14981f4ef704 Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Thu, 18 Dec 2025 14:03:28 +0100 Subject: [PATCH 2/8] Convert Build/Scripts/runTests.sh to v13 version --- Build/Scripts/runTests.sh | 475 ++++++++++++++++++++++++++++++-------- 1 file changed, 382 insertions(+), 93 deletions(-) diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh index f41559a..befc2b6 100755 --- a/Build/Scripts/runTests.sh +++ b/Build/Scripts/runTests.sh @@ -3,8 +3,12 @@ # # TYPO3 core test runner based on docker or podman # - -trap 'cleanUp;exit 2' SIGINT +# Copied from TYPO3 core: +# https://github.com/TYPO3/typo3/blob/v13.4.22/Build/Scripts/runTests.sh +# +if [ "${CI}" != "true" ]; then + trap 'echo "runTests.sh SIGINT signal emitted";cleanUp;exit 2' SIGINT +fi waitFor() { local HOST=${1} @@ -27,6 +31,7 @@ waitFor() { } cleanUp() { + echo "Remove container for network \"${NETWORK}\"" ATTACHED_CONTAINERS=$(${CONTAINER_BIN} ps --filter network=${NETWORK} --format='{{.Names}}') for ATTACHED_CONTAINER in ${ATTACHED_CONTAINERS}; do ${CONTAINER_BIN} kill ${ATTACHED_CONTAINER} >/dev/null @@ -162,17 +167,253 @@ cleanRenderedDocumentationFiles() { getPhpImageVersion() { case ${1} in 8.2) - echo -n "1.12" + echo -n "1.14" ;; 8.3) - echo -n "1.13" + echo -n "1.15" ;; 8.4) - echo -n "1.0" + echo -n "1.7" + ;; + 8.5) + echo -n "1.7" ;; esac } +# @todo: Add support for all available database engines (see -d option) +# @todo: Add support for classic mode +runPlaywright() { + PREPAREPARAMS="-e TYPO3_DB_DRIVER=sqlite" + TESTPARAMS="-e typo3DatabaseDriver=pdo_sqlite" + + if [ "${PLAYWRIGHT_USE_EXISTING_INSTANCE}x" = "x" ]; then + rm -rf "${CORE_ROOT}/typo3temp/var/tests/playwright-composer" "${CORE_ROOT}/typo3temp/var/tests/playwright-reports" "${CORE_ROOT}/typo3temp/var/tests/playwright-results" + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name playwright-prepare ${XDEBUG_MODE} -e COMPOSER_CACHE_DIR=${CORE_ROOT}/.cache/composer -e COMPOSER_ROOT_VERSION=${COMPOSER_ROOT_VERSION} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${PREPAREPARAMS} ${IMAGE_PHP} "${CORE_ROOT}/Build/Scripts/setupAcceptanceComposer.sh" "typo3temp/var/tests/playwright-composer" sqlite + if [[ $? -gt 0 ]]; then + kill -SIGINT -$$ + fi + fi + + [[ -e "${CORE_ROOT}/Build/node_modules/.bin/playwright" ]] || ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name playwright-${SUFFIX}-npm-ci \ + -e HOME=${CORE_ROOT}/.cache \ + ${IMAGE_NODEJS_CHROME} \ + npm --prefix=Build ci + if [[ $? -gt 0 ]]; then + kill -SIGINT -$$ + fi + + APACHE_OPTIONS="-e APACHE_RUN_USER=#${HOST_UID} -e APACHE_RUN_SERVERNAME=web -e APACHE_RUN_GROUP=#${HOST_PID} -e APACHE_RUN_DOCROOT=${CORE_ROOT}/typo3temp/var/tests/playwright-composer/public -e PHPFPM_HOST=phpfpm -e PHPFPM_PORT=9000" + if [[ ${PLAYWRIGHT_PREPARE_ONLY} -eq 1 ]]; then + APACHE_OPTIONS="${APACHE_OPTIONS} -p 127.0.0.1::80" + fi + + if [ ${CONTAINER_BIN} = "docker" ]; then + ${CONTAINER_BIN} run --rm -d --name ac-phpfpm-${SUFFIX} --network ${NETWORK} --network-alias phpfpm --add-host "${CONTAINER_HOST}:host-gateway" ${USERSET} -e PHPFPM_USER=${HOST_UID} -e PHPFPM_GROUP=${HOST_PID} -v ${CORE_ROOT}:${CORE_ROOT} ${IMAGE_PHP} php-fpm ${PHP_FPM_OPTIONS} >/dev/null + ${CONTAINER_BIN} run --rm -d --name ac-web-${SUFFIX} --network ${NETWORK} --network-alias web --add-host "${CONTAINER_HOST}:host-gateway" -v ${CORE_ROOT}:${CORE_ROOT} ${APACHE_OPTIONS} ${IMAGE_APACHE} >/dev/null + else + ${CONTAINER_BIN} run --rm ${CI_PARAMS} -d --name ac-phpfpm-${SUFFIX} --network ${NETWORK} --network-alias phpfpm ${USERSET} -e PHPFPM_USER=0 -e PHPFPM_GROUP=0 -v ${CORE_ROOT}:${CORE_ROOT} ${IMAGE_PHP} php-fpm -R ${PHP_FPM_OPTIONS} >/dev/null + ${CONTAINER_BIN} run --rm ${CI_PARAMS} -d --name ac-web-${SUFFIX} --network ${NETWORK} --network-alias web -v ${CORE_ROOT}:${CORE_ROOT} ${APACHE_OPTIONS} ${IMAGE_APACHE} >/dev/null + fi + + waitFor web 80 + + COMMAND="npm --prefix=${CORE_ROOT}/Build run playwright:run ${PLAYWRIGHT_PROJECT}" + COMMAND_UI="npm --prefix=${CORE_ROOT}/Build run playwright:open ${PLAYWRIGHT_PROJECT}" + if [[ ${PLAYWRIGHT_PREPARE_ONLY} -eq 0 ]]; then + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name accessibility-${SUFFIX} -e CHROME_SANDBOX=false -e CI=1 ${IMAGE_PLAYWRIGHT} ${COMMAND} + SUITE_EXIT_CODE=$? + else + PLAYWRIGHT_BASE_URL="http://$(${CONTAINER_BIN} port ac-web-${SUFFIX} 80/tcp)/" + echo + echo -en "\033[32m✓\033[0m " + echo "Environment prepared. You can now press Enter to run all tests or run playwright locally with one of the following commands." + echo + echo " Run with local playwright (headless):" + echo -n " " + echo "PLAYWRIGHT_BASE_URL=${PLAYWRIGHT_BASE_URL}typo3/ ${COMMAND}" + echo + echo " Open local playwright UI:" + echo -n " " + echo "PLAYWRIGHT_BASE_URL=${PLAYWRIGHT_BASE_URL}typo3/ ${COMMAND_UI}" + echo + echo -e "(Press \033[31mControl-C\033[0m to quit, \033[32mEnter\033[0m to run tests in container)" + # maybe use https://stackoverflow.com/a/58508884/4223467 + while read -r _; do + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name accessibility-${SUFFIX} -e CHROME_SANDBOX=false -e CI=1 ${IMAGE_PLAYWRIGHT} ${COMMAND} + SUITE_EXIT_CODE=$? + echo + echo -e "(Press \033[31mControl-C\033[0m to quit, \033[32mEnter\033[0m to re-run tests in container)" + done "${fullTargetFile}" + + echo -e "${GREEN} ✓ New file created${NC}" + echo "" + fi + + if [[ ! -f "${fullTargetFile}" ]]; then + echo -e "${RED}${fullTargetFile}${NC} could not be found and could not be created." + return 1 + fi + fi + else + echo -e "${YELLOW}HINT: File creation only works for EXT:core context." + echo -e "For other manuals, please create files distinctively, because the follow no pattern." + echo -e "Filename input is ignored.${NC}" + echo "" + fi + + echo -e "${YELLOW}NOTICE: Live documentation rendering is an experimental feature.${NC}" + echo " - Adding new files after the process is running will not include them" + echo " - Navigation / Menus on live-rendering may not fully work" + echo " - Leaving the process running for a long time may cause memory leaks / consumption" + echo "" + echo "After the initial rendering is done, you can access the local browser and edit the file" + echo "simultaneously. Every time the file is changed, your browser will automaticelly reload" + echo "the page, and scroll to the last position." + echo "" + + local htmlFile="${newFile%.rst}.html" + echo -e "Processing RST directory: ${GREEN}${systemExtensionFolder}/Documentation${NC}" + if [[ -f "${fullTargetFile}" ]]; then + echo -e "Working on: ${GREEN}${newFile}${NC}" + echo -e "Browser URL: ${GREEN}http://localhost:${actualRstPort}/${htmlFile}${NC}" + else + echo -e "Browser URL: ${GREEN}http://localhost:${actualRstPort}/${NC}" + fi + + echo -e "(Press ${RED}Control-C${NC} when finished writing documentation)" + echo "" + + # Command taken from Playwright example + if [ ${CONTAINER_BIN} = "docker" ]; then + ${CONTAINER_BIN} run -it --name watch-rst-rendering-${systemExtensionName}-${SUFFIX} -p ${actualRstPort}:${actualRstPort} --network ${NETWORK} --network-alias watch-rst --add-host "${CONTAINER_HOST}:host-gateway" -w /project -v "${CORE_ROOT}/${systemExtensionFolder}:/project" ${IMAGE_RSTRENDERING} --port ${actualRstPort} --watch --config=Documentation Documentation + else + ${CONTAINER_BIN} run -it ${CI_PARAMS} --name watch-rst-rendering-${systemExtensionName}-${SUFFIX} -p ${actualRstPort}:${actualRstPort} --network ${NETWORK} --network-alias watch-rst -w /project -v "${CORE_ROOT}/${systemExtensionFolder}:/project" ${IMAGE_RSTRENDERING} --port ${actualRstPort} --watch --config=Documentation Documentation + fi + + local exitCode=$? + echo "Render result for ${systemExtensionFolder}: ${exitCode}" + return ${exitCode} +} + loadHelp() { # Load help text into $HELP read -r -d '' HELP <" to use specific seed @@ -238,7 +483,7 @@ Options: - docker -a - Only with -s functional|functionalDeprecated + Only with -s functional Specifies to use another driver, following combinations are available: - mysql - mysqli (default) @@ -248,7 +493,7 @@ Options: - pdo_mysql -d - Only with -s functional|functionalDeprecated|acceptance|acceptanceComposer|acceptanceInstall + Only with -s functional|acceptance|acceptanceComposer|acceptanceInstall Specifies on which DBMS tests are performed - sqlite: (default): use sqlite - mariadb: use mariadb @@ -291,11 +536,12 @@ Options: Hack functional or acceptance tests into #numberOfChunks pieces and run tests of #chunk. Example -c 3/13 - -p <8.2|8.3|8.4> + -p <8.2|8.3|8.4|8.5> Specifies the PHP minor version to be used - 8.2 (default): use PHP 8.2 - 8.3: use PHP 8.3 - 8.4: use PHP 8.4 + - 8.5: use PHP 8.5 -t sets|systemplate Only with -s acceptance|acceptanceComposer @@ -309,7 +555,7 @@ Options: http://localhost:7900/. A browser tab is opened automatically if xdg-open is installed. -x - Only with -s functional|functionalDeprecated|unit|unitDeprecated|unitRandom|acceptance|acceptanceComposer|acceptanceInstall + Only with -s functional|unit|unitRandom|acceptance|acceptanceComposer|acceptanceInstall Send information to host instance for test or system under test break points. This is especially useful if a local PhpStorm instance is listening on default xdebug port 9003. A different port can be selected with -y @@ -376,8 +622,6 @@ if ! type "docker" >/dev/null 2>&1 && ! type "podman" >/dev/null 2>&1; then echo "This script relies on docker or podman. Please install" >&2 exit 1 fi -DBMS=mariadb -VERBOSE=1 # Go to the directory this script is located, so everything else is relative # to this dir, no matter from where this script is called, then go up two dirs. @@ -400,16 +644,22 @@ DATABASE_DRIVER="" CHUNKS=0 THISCHUNK=0 CONTAINER_BIN="" -COMPOSER_ROOT_VERSION="13.3.x-dev" +COMPOSER_ROOT_VERSION="13.4.x-dev" PHPSTAN_CONFIG_FILE="phpstan.local.neon" CONTAINER_INTERACTIVE="-it --init" HOST_UID=$(id -u) HOST_PID=$(id -g) USERSET="" +CI_PARAMS="${CI_PARAMS:-}" +CI_JOB_ID=${CI_JOB_ID:-} SUFFIX=$(echo $RANDOM) +if [ ${CI_JOB_ID} ]; then + SUFFIX="${CI_JOB_ID}-${SUFFIX}" +fi NETWORK="typo3-core-${SUFFIX}" -CI_PARAMS="${CI_PARAMS:-}" CONTAINER_HOST="host.docker.internal" +RST_TYPO3_MAIN_VERSION="13.4.x" +RST_PORT="1337" # Option parsing updates above default vars # Reset in case getopts has been used previously in the shell @@ -448,7 +698,7 @@ while getopts ":a:b:s:c:d:i:t:p:xy:nhug" OPT; do ;; p) PHP_VERSION=${OPTARG} - if ! [[ ${PHP_VERSION} =~ ^(8.1|8.2|8.3|8.4)$ ]]; then + if ! [[ ${PHP_VERSION} =~ ^(8.2|8.3|8.4|8.5)$ ]]; then INVALID_OPTIONS+=("${OPTARG}") fi ;; @@ -522,24 +772,22 @@ if ! type ${CONTAINER_BIN} >/dev/null 2>&1; then exit 1 fi -IMAGE_APACHE="ghcr.io/typo3/core-testing-apache24:1.5" +IMAGE_APACHE="ghcr.io/typo3/core-testing-apache24:1.7" IMAGE_PHP="ghcr.io/typo3/core-testing-$(echo "php${PHP_VERSION}" | sed -e 's/\.//'):$(getPhpImageVersion $PHP_VERSION)" -IMAGE_NODEJS="ghcr.io/typo3/core-testing-nodejs22:1.1" -IMAGE_NODEJS_CHROME="ghcr.io/typo3/core-testing-nodejs22-chrome:1.1" +IMAGE_NODEJS="ghcr.io/typo3/core-testing-nodejs22:1.3" +IMAGE_NODEJS_CHROME="ghcr.io/typo3/core-testing-nodejs22-chrome:1.3" +IMAGE_PLAYWRIGHT="mcr.microsoft.com/playwright:v1.56.1-noble" IMAGE_ALPINE="docker.io/alpine:3.8" -IMAGE_SELENIUM="docker.io/selenium/standalone-chrome:4.11.0-20230801" +# HEADS UP: We need to pin to <132 for --headless=old support until https://issues.chromium.org/issues/362522328 is resolved +IMAGE_SELENIUM="docker.io/selenium/standalone-chromium:131.0-20250101" IMAGE_REDIS="docker.io/redis:4-alpine" IMAGE_MEMCACHED="docker.io/memcached:1.5-alpine" IMAGE_MARIADB="docker.io/mariadb:${DBMS_VERSION}" IMAGE_MYSQL="docker.io/mysql:${DBMS_VERSION}" IMAGE_POSTGRES="docker.io/postgres:${DBMS_VERSION}-alpine" - -# Detect arm64 to use seleniarm image. -ARCH=$(uname -m) -if [ ${ARCH} = "arm64" ]; then - IMAGE_SELENIUM="docker.io/seleniarm/standalone-chromium:4.10.0-20230615" -fi +# Not a bug; render-guides has no "1.x" release yet. +IMAGE_RSTRENDERING="ghcr.io/typo3-documentation/render-guides:0.35" # Remove handled options and leaving the rest in the line, so it can be passed raw to commands shift $((OPTIND - 1)) @@ -559,6 +807,10 @@ else CONTAINER_COMMON_PARAMS="${CONTAINER_INTERACTIVE} ${CI_PARAMS} --rm --network ${NETWORK} -v ${CORE_ROOT}:${CORE_ROOT} -w ${CORE_ROOT}" fi +if [[ "${CI}" == "true" ]]; then + CONTAINER_COMMON_PARAMS="${CONTAINER_COMMON_PARAMS} ${CONTAINER_COMMON_PARAMS_CI:-}" +fi + if [ ${PHP_XDEBUG_ON} -eq 0 ]; then XDEBUG_MODE="-e XDEBUG_MODE=off" XDEBUG_CONFIG=" " @@ -578,9 +830,9 @@ case ${TEST_SUITE} in fi if [ "${CHUNKS}" -gt 0 ]; then ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name ac-splitter-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/splitAcceptanceTests.php -v ${CHUNKS} - COMMAND=(bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) + COMMAND=(php -d register_argc_argv=On bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) else - COMMAND=(bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) + COMMAND=(php -d register_argc_argv=On bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) fi SELENIUM_GRID="" if [ "${ACCEPTANCE_HEADLESS}" -eq 0 ]; then @@ -677,9 +929,9 @@ case ${TEST_SUITE} in fi if [ "${CHUNKS}" -gt 0 ]; then ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name ac-splitter-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/splitAcceptanceTests.php -v ${CHUNKS} - COMMAND=(bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) + COMMAND=(php -d register_argc_argv=On bin/codecept run Application -d -g AcceptanceTests-Job-${THISCHUNK} -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) else - COMMAND=(bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) + COMMAND=(php -d register_argc_argv=On bin/codecept run Application -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} "$@" --html reports.html) fi SELENIUM_GRID="" if [ "${ACCEPTANCE_HEADLESS}" -eq 0 ]; then @@ -744,7 +996,7 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name mariadb-ac-install-${SUFFIX} --network ${NETWORK} -d -e MYSQL_ROOT_PASSWORD=funcp --tmpfs /var/lib/mysql/:rw,noexec,nosuid ${IMAGE_MARIADB} >/dev/null waitFor mariadb-ac-install-${SUFFIX} 3306 CONTAINERPARAMS="-e typo3InstallMysqlDatabaseName=func_test -e typo3InstallMysqlDatabaseUsername=root -e typo3InstallMysqlDatabasePassword=funcp -e typo3InstallMysqlDatabaseHost=mariadb-ac-install-${SUFFIX}" - COMMAND="bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" + COMMAND="php -d register_argc_argv=On bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name ac-install-mariadb ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} ${COMMAND} SUITE_EXIT_CODE=$? ;; @@ -756,7 +1008,7 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name mysql-ac-install-${SUFFIX} --network ${NETWORK} -d -e MYSQL_ROOT_PASSWORD=funcp --tmpfs /var/lib/mysql/:rw,noexec,nosuid ${IMAGE_MYSQL} >/dev/null waitFor mysql-ac-install-${SUFFIX} 3306 CONTAINERPARAMS="-e typo3InstallMysqlDatabaseName=func_test -e typo3InstallMysqlDatabaseUsername=root -e typo3InstallMysqlDatabasePassword=funcp -e typo3InstallMysqlDatabaseHost=mysql-ac-install-${SUFFIX}" - COMMAND="bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" + COMMAND="php -d register_argc_argv=On bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name ac-install-mysql ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} ${COMMAND} SUITE_EXIT_CODE=$? ;; @@ -768,7 +1020,7 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name postgres-ac-install-${SUFFIX} --network ${NETWORK} -d -e POSTGRES_PASSWORD=funcp -e POSTGRES_USER=funcu --tmpfs /var/lib/postgresql/data:rw,noexec,nosuid ${IMAGE_POSTGRES} >/dev/null waitFor postgres-ac-install-${SUFFIX} 5432 CONTAINERPARAMS="-e typo3InstallPostgresqlDatabasePort=5432 -e typo3InstallPostgresqlDatabaseName=${USER} -e typo3InstallPostgresqlDatabaseHost=postgres-ac-install-${SUFFIX} -e typo3InstallPostgresqlDatabaseUsername=funcu -e typo3InstallPostgresqlDatabasePassword=funcp" - COMMAND="bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" + COMMAND="php -d register_argc_argv=On bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name ac-install-postgres ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} ${COMMAND} SUITE_EXIT_CODE=$? ;; @@ -780,20 +1032,35 @@ case ${TEST_SUITE} in CODECEPION_ENV="--env ci,sqlite,headless" fi CONTAINERPARAMS="-e typo3DatabaseDriver=pdo_sqlite" - COMMAND="bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" + COMMAND="php -d register_argc_argv=On bin/codecept run Install -d -c typo3/sysext/core/Tests/codeception.yml ${CODECEPION_ENV} --html reports.html" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name ac-install-sqlite ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} ${COMMAND} SUITE_EXIT_CODE=$? ;; esac ;; - buildCss) - COMMAND="cd Build; npm ci || exit 1; node_modules/grunt/bin/grunt css" - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name build-css-${SUFFIX} -e HOME=${CORE_ROOT}/.cache ${IMAGE_NODEJS} /bin/sh -c "${COMMAND}" - SUITE_EXIT_CODE=$? + e2e) + PLAYWRIGHT_PROJECT="--project e2e" + PLAYWRIGHT_PREPARE_ONLY=0 + runPlaywright ;; - buildJavascript) - COMMAND="cd Build/; npm ci || exit 1; node_modules/grunt/bin/grunt scripts" - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name build-js-${SUFFIX} -e HOME=${CORE_ROOT}/.cache ${IMAGE_NODEJS} /bin/sh -c "${COMMAND}" + e2e-prepare) + PLAYWRIGHT_PROJECT="--project e2e" + PLAYWRIGHT_PREPARE_ONLY=1 + runPlaywright + ;; + accessibility) + PLAYWRIGHT_PROJECT="--project accessibility" + PLAYWRIGHT_PREPARE_ONLY=0 + runPlaywright + ;; + accessibility-prepare) + PLAYWRIGHT_PROJECT="--project accessibility" + PLAYWRIGHT_PREPARE_ONLY=1 + runPlaywright + ;; + build*) + COMMAND="cd Build; npm install && npm run build" + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name build-css-${SUFFIX} -e HOME=${CORE_ROOT}/.cache ${IMAGE_NODEJS} /bin/sh -c "${COMMAND}" SUITE_EXIT_CODE=$? ;; cgl) @@ -830,6 +1097,10 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-utf8bom-${SUFFIX} ${IMAGE_PHP} Build/Scripts/checkUtf8Bom.sh SUITE_EXIT_CODE=$? ;; + checkIntegritySetLabels) + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-integrity-set-labels-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/checkIntegritySetLabels.php + SUITE_EXIT_CODE=$? + ;; checkComposer) ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-composer-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/checkIntegrityComposer.php SUITE_EXIT_CODE=$? @@ -842,6 +1113,10 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-file-path-length-${SUFFIX} ${IMAGE_PHP} Build/Scripts/maxFilePathLength.sh SUITE_EXIT_CODE=$? ;; + checkFilesAndPathsForSpaces) + ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-file-path-length-${SUFFIX} ${IMAGE_PHP} Build/Scripts/spacesInPathsAndFilenames.sh + SUITE_EXIT_CODE=$? + ;; checkGitSubmodule) COMMAND="if [ \$(git submodule status 2>&1 | wc -l) -ne 0 ]; then echo \"Found a submodule definition in repository\"; exit 1; fi" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-git-submodule-${SUFFIX} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" @@ -865,6 +1140,55 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name check-rst-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/validateRstFiles.php SUITE_EXIT_CODE=$? ;; + checkRstRenderingAll) + SUITE_EXIT_CODE=0 + echo "Scanning typo3/sysext for directories with Documentation..." + for systemExtensionFolder in typo3/sysext/*/Documentation; do + systemExtensionName="${systemExtensionFolder%/Documentation}" + systemExtensionName=$(basename "${systemExtensionName}") + executeRstRendering "${systemExtensionName}" + TMP_SUITE_EXIT_CODE=$? + if [ ${TMP_SUITE_EXIT_CODE} -ne 0 ]; then + SUITE_EXIT_CODE=${TMP_SUITE_EXIT_CODE} + fi + done + ;; + checkRstRenderingSingle) + systemExtensionKey="${1}" + if [ -n "${systemExtensionKey}" ]; then + if [[ ! -d "typo3/sysext/${systemExtensionKey}" ]]; then + echo "Error: Invalid system extension key provided: \"${systemExtensionKey}\"" + SUITE_EXIT_CODE=1 + elif [[ ! -d "typo3/sysext/${systemExtensionKey}/Documentation" ]]; then + echo "Error: Valid system extension \"${systemExtensionKey}\" does not contain a \"Documentation\" folder" + SUITE_EXIT_CODE=1 + else + executeRstRendering "${systemExtensionKey}" + SUITE_EXIT_CODE=$? + fi + else + echo "Error: No system extension key provided as first argument" + SUITE_EXIT_CODE=1 + fi + ;; + watchRst) + systemExtensionKey="${1}" + if [ -n "${systemExtensionKey}" ]; then + if [[ ! -d "typo3/sysext/${systemExtensionKey}" ]]; then + echo "Error: Invalid system extension key provided: \"${systemExtensionKey}\"" + SUITE_EXIT_CODE=1 + elif [[ ! -d "typo3/sysext/${systemExtensionKey}/Documentation" ]]; then + echo "Error: Valid system extension \"${systemExtensionKey}\" does not contain a \"Documentation\" folder" + SUITE_EXIT_CODE=1 + else + executeRstRenderingWithWatch "${systemExtensionKey}" "${2}" + SUITE_EXIT_CODE=$? + fi + else + echo "Error: No system extension key provided as first argument" + SUITE_EXIT_CODE=1 + fi + ;; clean) cleanBuildFiles cleanCacheFiles @@ -915,9 +1239,9 @@ case ${TEST_SUITE} in functional) if [ "${CHUNKS}" -gt 0 ]; then ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name func-splitter-${SUFFIX} ${IMAGE_PHP} php -dxdebug.mode=off Build/Scripts/splitFunctionalTests.php -v ${CHUNKS} - COMMAND=(.Build/bin/phpunit -c Build/phpunit/FunctionalTests-Job-${THISCHUNK}.xml --exclude-group not-${DBMS} "$@") + COMMAND=(bin/phpunit -c Build/phpunit/FunctionalTests-Job-${THISCHUNK}.xml --exclude-group not-${DBMS} "$@") else - COMMAND=(.Build/bin/phpunit -c Build/phpunit/FunctionalTests.xml --exclude-group not-${DBMS} "$@") + COMMAND=(bin/phpunit -c Build/phpunit/FunctionalTests.xml --exclude-group not-${DBMS} "$@") fi ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name redis-func-${SUFFIX} --network ${NETWORK} -d ${IMAGE_REDIS} >/dev/null ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name memcached-func-${SUFFIX} --network ${NETWORK} -d ${IMAGE_MEMCACHED} >/dev/null @@ -957,46 +1281,6 @@ case ${TEST_SUITE} in ;; esac ;; - functionalDeprecated) - COMMAND=(bin/phpunit -c Build/phpunit/FunctionalTestsDeprecated.xml --exclude-group not-${DBMS} "$@") - ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name redis-func-dep-${SUFFIX} --network ${NETWORK} -d ${IMAGE_REDIS} >/dev/null - ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name memcached-func-dep-${SUFFIX} --network ${NETWORK} -d ${IMAGE_MEMCACHED} >/dev/null - waitFor redis-func-dep-${SUFFIX} 6379 - waitFor memcached-func-dep-${SUFFIX} 11211 - CONTAINER_COMMON_PARAMS="${CONTAINER_COMMON_PARAMS} -e typo3TestingRedisHost=redis-func-dep-${SUFFIX} -e typo3TestingMemcachedHost=memcached-func-dep-${SUFFIX}" - case ${DBMS} in - mariadb) - echo "Using driver: ${DATABASE_DRIVER}" - ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name mariadb-func-dep-${SUFFIX} --network ${NETWORK} -d -e MYSQL_ROOT_PASSWORD=funcp --tmpfs /var/lib/mysql/:rw,noexec,nosuid ${IMAGE_MARIADB} >/dev/null - waitFor mariadb-func-dep-${SUFFIX} 3306 - CONTAINERPARAMS="-e typo3DatabaseDriver=${DATABASE_DRIVER} -e typo3DatabaseName=func_test -e typo3DatabaseUsername=root -e typo3DatabaseHost=mariadb-func-dep-${SUFFIX} -e typo3DatabasePassword=funcp" - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name functional-deprecated-${SUFFIX} ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} "${COMMAND[@]}" - SUITE_EXIT_CODE=$? - ;; - mysql) - echo "Using driver: ${DATABASE_DRIVER}" - ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name mysql-func-dep-${SUFFIX} --network ${NETWORK} -d -e MYSQL_ROOT_PASSWORD=funcp --tmpfs /var/lib/mysql/:rw,noexec,nosuid ${IMAGE_MYSQL} >/dev/null - waitFor mysql-func-dep-${SUFFIX} 3306 - CONTAINERPARAMS="-e typo3DatabaseDriver=${DATABASE_DRIVER} -e typo3DatabaseName=func_test -e typo3DatabaseUsername=root -e typo3DatabaseHost=mysql-func-dep-${SUFFIX} -e typo3DatabasePassword=funcp" - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name functional-deprecated-${SUFFIX} ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} "${COMMAND[@]}" - SUITE_EXIT_CODE=$? - ;; - postgres) - ${CONTAINER_BIN} run --rm ${CI_PARAMS} --name postgres-func-dep-${SUFFIX} --network ${NETWORK} -d -e POSTGRES_PASSWORD=funcp -e POSTGRES_USER=funcu --tmpfs /var/lib/postgresql/data:rw,noexec,nosuid ${IMAGE_POSTGRES} >/dev/null - waitFor postgres-func-dep-${SUFFIX} 5432 - CONTAINERPARAMS="-e typo3DatabaseDriver=pdo_pgsql -e typo3DatabaseName=bamboo -e typo3DatabaseUsername=funcu -e typo3DatabaseHost=postgres-func-dep-${SUFFIX} -e typo3DatabasePassword=funcp" - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name functional-deprecated-${SUFFIX} ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} "${COMMAND[@]}" - SUITE_EXIT_CODE=$? - ;; - sqlite) - # create sqlite tmpfs mount typo3temp/var/tests/functional-sqlite-dbs/ to avoid permission issues - mkdir -p "${CORE_ROOT}/typo3temp/var/tests/functional-sqlite-dbs/" - CONTAINERPARAMS="-e typo3DatabaseDriver=pdo_sqlite --tmpfs ${CORE_ROOT}/typo3temp/var/tests/functional-sqlite-dbs/:rw,noexec,nosuid" - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name functional-deprecated-${SUFFIX} ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${CONTAINERPARAMS} ${IMAGE_PHP} "${COMMAND[@]}" - SUITE_EXIT_CODE=$? - ;; - esac - ;; listExceptionCodes) ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name list-exception-codes-${SUFFIX} ${IMAGE_PHP} Build/Scripts/duplicateExceptionCodeCheck.sh -p SUITE_EXIT_CODE=$? @@ -1027,7 +1311,7 @@ case ${TEST_SUITE} in SUITE_EXIT_CODE=$? ;; lintYaml) - EXCLUDE_INVALID_FIXTURE_YAML_FILES="--exclude typo3/sysext/form/Tests/Unit/Mvc/Configuration/Fixtures/Invalid.yaml" + EXCLUDE_INVALID_FIXTURE_YAML_FILES="--exclude typo3/sysext/form/Tests/Unit/Mvc/Configuration/Fixtures/Invalid.yaml --exclude typo3/sysext/core/Tests/Functional/Fixtures/Extensions/test_sets/Configuration/Sets/InvalidSettings/settings.yaml --exclude typo3/sysext/core/Tests/Functional/Configuration/Loader/Fixtures/InvalidYamlFiles/LoadEmptyYaml.yaml --exclude typo3/sysext/core/Tests/Functional/Configuration/Loader/Fixtures/InvalidYamlFiles/LoadInvalidYaml.yaml" COMMAND="php -v | grep '^PHP'; find typo3/ \\( -name '*.yaml' -o -name '*.yml' \\) ! -name 'Services.yaml' | xargs -r php -dxdebug.mode=off bin/yaml-lint --no-parse-tags ${EXCLUDE_INVALID_FIXTURE_YAML_FILES}" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name lint-php-${SUFFIX} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" SUITE_EXIT_CODE=$? @@ -1038,12 +1322,12 @@ case ${TEST_SUITE} in SUITE_EXIT_CODE=$? ;; phpstan) - COMMAND=(php -dxdebug.mode=off bin/phpstan analyse -c Build/phpstan/${PHPSTAN_CONFIG_FILE} --no-progress --no-interaction --memory-limit 4G "$@") + COMMAND=(php -dxdebug.mode=off bin/phpstan analyse -c Build/phpstan/${PHPSTAN_CONFIG_FILE} --verbose --no-progress --no-interaction --memory-limit 4G "$@") ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name phpstan-${SUFFIX} ${IMAGE_PHP} "${COMMAND[@]}" SUITE_EXIT_CODE=$? ;; phpstanGenerateBaseline) - COMMAND="php -dxdebug.mode=off bin/phpstan analyse -c Build/phpstan/${PHPSTAN_CONFIG_FILE} --no-progress --no-interaction --memory-limit 4G --generate-baseline=Build/phpstan/phpstan-baseline.neon" + COMMAND="php -dxdebug.mode=off bin/phpstan analyse -c Build/phpstan/${PHPSTAN_CONFIG_FILE} --verbose --no-progress --no-interaction --memory-limit 4G --generate-baseline=Build/phpstan/phpstan-baseline.neon" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name phpstan-baseline-${SUFFIX} ${IMAGE_PHP} /bin/sh -c "${COMMAND}" SUITE_EXIT_CODE=$? ;; @@ -1051,10 +1335,6 @@ case ${TEST_SUITE} in ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name unit-${SUFFIX} ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${IMAGE_PHP} bin/phpunit -c Build/phpunit/UnitTests.xml "$@" SUITE_EXIT_CODE=$? ;; - unitDeprecated) - ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name unit-deprecated-${SUFFIX} ${XDEBUG_MODE} -e XDEBUG_CONFIG="${XDEBUG_CONFIG}" ${IMAGE_PHP} bin/phpunit -c Build/phpunit/UnitTestsDeprecated.xml "$@" - SUITE_EXIT_CODE=$? - ;; unitJavascript) COMMAND="cd Build; npm ci || exit 1; CHROME_SANDBOX=false BROWSERS=chrome npm run test" ${CONTAINER_BIN} run ${CONTAINER_COMMON_PARAMS} --name unit-javascript-${SUFFIX} -e HOME=${CORE_ROOT}/.cache ${IMAGE_NODEJS_CHROME} /bin/sh -c "${COMMAND}" @@ -1073,6 +1353,14 @@ case ${TEST_SUITE} in echo "> remove \"dangling\" ghcr.io/typo3/core-testing-* images (those tagged as )" ${CONTAINER_BIN} images --filter "reference=ghcr.io/typo3/core-testing-*" --filter "dangling=true" --format "{{.ID}}" | xargs -I {} ${CONTAINER_BIN} rmi -f {} echo "" + # pull ghcr.io/typo3-documentation/render-guides versions of those ones that exist locally + echo "> pull ghcr.io/typo3-documentation/render-guides versions of those ones that exist locally" + ${CONTAINER_BIN} images "ghcr.io/typo3-documentation/render-guides" --format "{{.Repository}}:{{.Tag}}" | xargs -I {} ${CONTAINER_BIN} pull {} + echo "" + # remove "dangling" ghcr.io/typo3-documentation/render-guides* images (those tagged as ) + echo "> remove \"dangling\" ghcr.io/typo3-documentation/render-guides images (those tagged as )" + ${CONTAINER_BIN} images --filter "reference=ghcr.io/typo3-documentation/render-guides" --filter "dangling=true" --format "{{.ID}}" | xargs -I {} ${CONTAINER_BIN} rmi -f {} + echo "" ;; *) loadHelp @@ -1090,8 +1378,9 @@ echo "" >&2 echo "###########################################################################" >&2 echo "Result of ${TEST_SUITE}" >&2 echo "Container runtime: ${CONTAINER_BIN}" >&2 +echo "Container suffix: ${SUFFIX}" echo "PHP: ${PHP_VERSION}" >&2 -if [[ ${TEST_SUITE} =~ ^(functional|functionalDeprecated|acceptance|acceptanceComposer|acceptanceInstall)$ ]]; then +if [[ ${TEST_SUITE} =~ ^(functional|acceptance|acceptanceComposer|acceptanceInstall)$ ]]; then case "${DBMS}" in mariadb|mysql|postgres) echo "DBMS: ${DBMS} version ${DBMS_VERSION} driver ${DATABASE_DRIVER}" >&2 From a92002c8001eecf2540b33568c19c7c6ba5159bb Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Thu, 18 Dec 2025 15:38:34 +0100 Subject: [PATCH 3/8] Adjust config files to get new version of runTests.sh working --- .gitignore | 4 ++++ Tests/Functional/Command/UnduplicateCommandTest.php | 4 ++-- composer.json | 3 +-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 859dc90..5614da7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,13 @@ /.php_cs.cache /.php-cs-fixer.cache /Build/testing-docker/.env +/Build/phpunit/FunctionalTests-Job-* +/Build/phpunit/.phpunit.cache /composer.lock /Documentation-GENERATED-temp/ /public/ /Tests/Unit/.phpunit.result.cache /var/ /vendor/ +/bin/ +/typo3temp diff --git a/Tests/Functional/Command/UnduplicateCommandTest.php b/Tests/Functional/Command/UnduplicateCommandTest.php index d6dd076..e0e250e 100644 --- a/Tests/Functional/Command/UnduplicateCommandTest.php +++ b/Tests/Functional/Command/UnduplicateCommandTest.php @@ -182,10 +182,10 @@ class UnduplicateCommandTest extends FunctionalTestCase */ protected function executeConsoleCommand(string $cmdline, ...$args): array { - $typo3File = __DIR__ . '/../../../.Build/bin/typo3'; + $typo3File = __DIR__ . '/../../../bin/typo3'; if (!file_exists($typo3File)) { throw new RuntimeException( - sprintf('Executable file not found (using path <%s>). Make sure config:bin-dir is set to .Build/bin in composer.json', $typo3File) + sprintf('Executable file not found (using path <%s>). Make sure config:bin-dir is set to \'bin\' in composer.json', $typo3File) ); } diff --git a/composer.json b/composer.json index e7041b9..1e4d9cf 100644 --- a/composer.json +++ b/composer.json @@ -26,8 +26,7 @@ }, "config": { "sort-packages": true, - "vendor-dir": ".Build/vendor", - "bin-dir": ".Build/bin", + "bin-dir": "bin", "allow-plugins": { "typo3/cms-composer-installers": true, "typo3/class-alias-loader": true From d84233bd6d0811d2b219248e638315039b7da342 Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Thu, 18 Dec 2025 14:03:55 +0100 Subject: [PATCH 4/8] Update github workflow --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6f78b6e..6af326b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,19 +13,19 @@ on: jobs: - all_core_12: - name: "all core-12" - runs-on: ubuntu-22.04 + all_core_13: + name: "all core-13" + runs-on: ubuntu-24.04 strategy: # This prevents cancellation of matrix job runs, if one/two already failed and let the # rest of the matrix jobs be executed anyway. fail-fast: false matrix: - php: [ '8.2', '8.3', '8.4' ] + php: [ '8.2', '8.3', '8.4', '8.5' ] minMax: [ 'composerInstall'] steps: - name: "Checkout" - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: "Composer" run: Build/Scripts/runTests.sh -p ${{ matrix.php }} -s ${{ matrix.minMax }} From 9598b704724e2c0fa7b856ebbf2732560a4f1f73 Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Thu, 18 Dec 2025 16:46:36 +0100 Subject: [PATCH 5/8] Run rector + cs-fixer for v13 compatibility + manual fixes --- Build/phpunit/FunctionalTestsBootstrap.php | 2 +- Classes/Command/UnduplicateCommand.php | 329 ++++++++---------- Classes/Metadata/MetadataUpdateHandler.php | 115 +++--- Classes/Metadata/MetadataUpdateObject.php | 4 +- .../Command/UnduplicateCommandTest.php | 12 +- 5 files changed, 196 insertions(+), 266 deletions(-) diff --git a/Build/phpunit/FunctionalTestsBootstrap.php b/Build/phpunit/FunctionalTestsBootstrap.php index 6945270..73e43ac 100644 --- a/Build/phpunit/FunctionalTestsBootstrap.php +++ b/Build/phpunit/FunctionalTestsBootstrap.php @@ -24,7 +24,7 @@ * This file is defined in FunctionalTests.xml and called by phpunit * before instantiating the test suites. */ -(static function () { +(static function (): void { $testbase = new Testbase(); $testbase->defineOriginalRootPath(); $testbase->createDirectory(ORIGINAL_ROOT . 'typo3temp/var/tests'); diff --git a/Classes/Command/UnduplicateCommand.php b/Classes/Command/UnduplicateCommand.php index c36f959..6a96ef2 100644 --- a/Classes/Command/UnduplicateCommand.php +++ b/Classes/Command/UnduplicateCommand.php @@ -59,7 +59,6 @@ */ class UnduplicateCommand extends Command { - /** * @var bool */ @@ -70,114 +69,96 @@ class UnduplicateCommand extends Command */ private $keepOldest = false; - /** - * @var int - */ - private $storage = -1; + private int $storage = -1; - /** - * @var SymfonyStyle - */ - private $output; + private ?SymfonyStyle $output = null; - /** - * @var InputInterface - */ - private $input; + private ?InputInterface $input = null; - /** - * @var array - */ - private $fieldsToCheck = ['description']; + private array $fieldsToCheck = ['description']; - /** - * @var MetadataUpdateHandler - */ - private $metadataHandler; + private ?MetadataUpdateHandler $metadataHandler = null; public function __construct( - private readonly ConnectionPool $connectionPool, + private readonly ConnectionPool $connectionPool, private readonly StorageRepository $storageRepository - ) - { + ) { parent::__construct(); } /** * Configure the command by defining the name, options and arguments */ - public function configure() + public function configure(): void { - $this->setDescription("Finds duplicates in sys_file and unduplicates them. - By default it will use the newest (highest uid) record as master and delete the older records."); + $this->setDescription('Finds duplicates in sys_file and unduplicates them. + By default it will use the newest (highest uid) record as master and delete the older records.'); $this->setHelp( - "currently fix references in " . LF . - "- sys_file_reference::link " . LF . - "- sys_file_reference::uid_local " . LF . - "- tt_content::headerlink " . LF . - "- tt_content::bodytext " . LF . - "- tx_news_domain_model_news::bodytext " . LF . - "- tx_news_domain_model_news::internalurl " . LF . - "AND the remove the duplicate in sys_file and sys_file_metadata" + 'currently fix references in ' . LF . + '- sys_file_reference::link ' . LF . + '- sys_file_reference::uid_local ' . LF . + '- tt_content::headerlink ' . LF . + '- tt_content::bodytext ' . LF . + '- tx_news_domain_model_news::bodytext ' . LF . + '- tx_news_domain_model_news::internalurl ' . LF . + 'AND the remove the duplicate in sys_file and sys_file_metadata' ); $this->addOption( - "dry-run", - "d", + 'dry-run', + 'd', InputOption::VALUE_NONE, - "If set, all database updates are not executed" + 'If set, all database updates are not executed' ) ->addOption( - "identifier", - "i", + 'identifier', + 'i', InputOption::VALUE_REQUIRED, - "Only use this identifier" + 'Only use this identifier' ) ->addOption( - "storage", - "s", + 'storage', + 's', InputOption::VALUE_REQUIRED, - "Only use this storage", + 'Only use this storage', -1 ) ->addOption( - "force", - "f", + 'force', + 'f', InputOption::VALUE_OPTIONAL, - "Force keep or overwrite of metadata of the master record in case of conflict. Possible values: keep, keep-nonempty. - Default: overwrite. Keep-nonempty is keeping only nonempty metadata in master, but updating the empty.", + 'Force keep or overwrite of metadata of the master record in case of conflict. Possible values: keep, keep-nonempty. + Default: overwrite. Keep-nonempty is keeping only nonempty metadata in master, but updating the empty.', false ) ->addOption( - "keep-oldest", - "o", + 'keep-oldest', + 'o', InputOption::VALUE_NONE, - "Use the oldest record as master instead of the newest", + 'Use the oldest record as master instead of the newest', ) ->addOption( - "interactive", - "a", + 'interactive', + 'a', InputOption::VALUE_NONE, - "When encountering a conflict, ask for user input to determine which record should be kept.", + 'When encountering a conflict, ask for user input to determine which record should be kept.', ) ->addOption( - "meta-fields", - "m", + 'meta-fields', + 'm', InputOption::VALUE_REQUIRED, - "Specify a comma seperated list of fields to check for metadata comparison. - Default description, with EXT:filemetadata: description, caption, copyright" + 'Specify a comma seperated list of fields to check for metadata comparison. + Default description, with EXT:filemetadata: description, caption, copyright' )->addOption( - "update-refindex", - "u", + 'update-refindex', + 'u', InputOption::VALUE_NONE, - "Setting this option automatically updates the reference index and does not ask on command line. - Alternatively, use -n to avoid the interactive mode" + 'Setting this option automatically updates the reference index and does not ask on command line. + Alternatively, use -n to avoid the interactive mode' ); } /** * make sure we do not have duplicate records with same language - * @param array $oldMetadataRecords - * @return mixed * @throws UnduplicatorException */ public function checkDuplicateMetadataRecords(array $oldMetadataRecords): void @@ -185,7 +166,7 @@ public function checkDuplicateMetadataRecords(array $oldMetadataRecords): void $languageRecords = []; foreach ($oldMetadataRecords as $oldMetadata) { if (isset($languageRecords[$oldMetadata['sys_language_uid']])) { - throw new UnduplicatorException("More than one metadata record for language " . $oldMetadata['sys_language_uid']); + throw new UnduplicatorException('More than one metadata record for language ' . $oldMetadata['sys_language_uid'], 7813804023); } $languageRecords[$oldMetadata['sys_language_uid']][] = $oldMetadata; } @@ -194,8 +175,6 @@ public function checkDuplicateMetadataRecords(array $oldMetadataRecords): void /** * Executes the command * - * @param InputInterface $input - * @param OutputInterface $output * @throws Exception */ protected function execute(InputInterface $input, OutputInterface $output): int @@ -207,34 +186,31 @@ protected function execute(InputInterface $input, OutputInterface $output): int // Update the reference index $this->updateReferenceIndex($input); - $this->dryRun = $input->getOption("dry-run"); - $this->keepOldest = $input->getOption("keep-oldest"); - $onlyThisIdentifier = $input->getOption("identifier"); - $this->storage = (int)$input->getOption("storage") ?? -1; + $this->dryRun = $input->getOption('dry-run'); + $this->keepOldest = $input->getOption('keep-oldest'); + $onlyThisIdentifier = $input->getOption('identifier'); + $this->storage = (int)$input->getOption('storage') ?? -1; - if ($input->hasArgument("meta-fields")) { - $this->fieldsToCheck = array_map("trim", explode(",", $input->getOption("meta-fields"))); - } elseif (ExtensionManagementUtility::isLoaded("filemetadata")) { + if ($input->hasArgument('meta-fields')) { + $this->fieldsToCheck = array_map(trim(...), explode(',', (string)$input->getOption('meta-fields'))); + } elseif (ExtensionManagementUtility::isLoaded('filemetadata')) { // add default values in case the filemetadata extension is loaded $this->fieldsToCheck = array_merge($this->fieldsToCheck, ['caption", "copyright']); } - $this->output->writeln("Using metadata fields: " . implode(", ", $this->fieldsToCheck) . ""); + $this->output->writeln('Using metadata fields: ' . implode(', ', $this->fieldsToCheck) . ''); $this->metadataHandler = new MetadataUpdateHandler($this->dryRun, $this->input, $this->output, $this->fieldsToCheck, $this->connectionPool); try { $this->runOn($onlyThisIdentifier); } catch (UnduplicatorException $e) { - $this->output->writeln("" . $e->getMessage() . ""); + $this->output->writeln('' . $e->getMessage() . ''); } return 0; } /** - * @param mixed $onlyThisIdentifier - * @param int $onlyThisStorage - * @return void * @throws Exception */ public function runOn(mixed $onlyThisIdentifier): void @@ -243,9 +219,9 @@ public function runOn(mixed $onlyThisIdentifier): void $statement = $this->findDuplicates($onlyThisIdentifier); $foundDuplicates = 0; while ($row = $statement->fetchAssociative()) { - $identifier = $row['identifier'] ?? ""; - if ($identifier === "") { - $this->output->warning("Found empty identifier"); + $identifier = $row['identifier'] ?? ''; + if ($identifier === '') { + $this->output->warning('Found empty identifier'); continue; } @@ -275,10 +251,10 @@ public function runOn(mixed $onlyThisIdentifier): void } } if (!$foundDuplicates) { - $this->output->success("No duplicates found"); + $this->output->success('No duplicates found'); } if ($hasConflicts) { - $this->output->warning("Conflicts found. Manual action required. Run with -v to see details."); + $this->output->warning('Conflicts found. Manual action required. Run with -v to see details.'); } } @@ -287,27 +263,23 @@ public function runOn(mixed $onlyThisIdentifier): void * if checked case-insensitively. * * Database may be case-insensitive, e.g. charset "utf8mb5", collation "utf8mb4_unicode_ci". - * - * @param mixed $onlyThisIdentifier - * @param int $storage - * @return Result */ public function findDuplicates(mixed $onlyThisIdentifier): Result { - $queryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_file"); - $queryBuilder->count("*") - ->from("sys_file") - ->having("COUNT(*) > 1"); + $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file'); + $queryBuilder->count('*') + ->from('sys_file') + ->having('COUNT(*) > 1'); $whereExpressions = []; if ($onlyThisIdentifier) { $whereExpressions[] = $queryBuilder->expr()->eq( - "identifier", - $queryBuilder->createNamedParameter($onlyThisIdentifier, \PDO::PARAM_STR) + 'identifier', + $queryBuilder->createNamedParameter($onlyThisIdentifier, Connection::PARAM_STR) ); } if ($this->storage > -1) { $whereExpressions[] = $queryBuilder->expr()->eq( - "storage", + 'storage', $queryBuilder->createNamedParameter($this->storage, Connection::PARAM_INT) ); } @@ -318,18 +290,16 @@ public function findDuplicates(mixed $onlyThisIdentifier): Result $concreteQueryBuilder = $queryBuilder->getConcreteQueryBuilder(); // GROUP BY BINARY `identifier`,`storage - $concreteQueryBuilder->groupBy("MD5(identifier)"); - $concreteQueryBuilder->addGroupBy("storage"); + $concreteQueryBuilder->groupBy('MD5(identifier)'); + $concreteQueryBuilder->addGroupBy('storage'); // SELECT MAX(`identifier`) AS identifier,`storage` - $concreteQueryBuilder->addSelect("MAX(identifier) AS identifier, storage"); + $concreteQueryBuilder->addSelect('MAX(identifier) AS identifier, storage'); if ($this->output->isDebug()) { - $this->output->writeln("sql=" . $queryBuilder->getSQL(), OutputInterface::VERBOSITY_VERBOSE); + $this->output->writeln('sql=' . $queryBuilder->getSQL(), OutputInterface::VERBOSITY_VERBOSE); } - - $statement = $queryBuilder + return $queryBuilder ->executeQuery(); - return $statement; } /** @@ -339,35 +309,36 @@ public function findDuplicates(mixed $onlyThisIdentifier): Result */ private function findDuplicateFilesForIdentifier(string $identifier, int $storage): array { - $fileQueryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_file"); + $fileQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file'); - $fileQueryBuilder->select("uid", "identifier") - ->from("sys_file") + $fileQueryBuilder->select('uid', 'identifier') + ->from('sys_file') ->where( $fileQueryBuilder->expr()->eq( - "storage", + 'storage', $fileQueryBuilder->createNamedParameter($storage, Connection::PARAM_INT) ) - )->orderBy("uid", $this->keepOldest ? "ASC" : "DESC"); + )->orderBy('uid', $this->keepOldest ? 'ASC' : 'DESC'); - $whereClause = "MD5(identifier) = MD5(" . $fileQueryBuilder->createNamedParameter($identifier, \PDO::PARAM_STR) . ")"; - $fileQueryBuilder->add("where", $whereClause); + $whereClause = 'MD5(identifier) = MD5(' . $fileQueryBuilder->createNamedParameter($identifier, Connection::PARAM_STR) . ')'; + $fileQueryBuilder->add('where', $whereClause); return $fileQueryBuilder->executeQuery() ->fetchAllAssociative(); } /** - * @param int $masterFileUid - * @param int $oldFileUid - * @param mixed $identifier - * @param int $storage * @throws Exception */ - public function processDuplicate(int $masterFileUid, array $masterMetadataRecords, int $oldFileUid, mixed $identifier, int $storage): bool - { + public function processDuplicate( + int $masterFileUid, + array $masterMetadataRecords, + int $oldFileUid, + mixed $identifier, + int $storage + ): bool { $this->output->writeln(sprintf( - "Unduplicate sys_file: uid=%d identifier=\"%s\", storage=%s (master uid=%d)", + 'Unduplicate sys_file: uid=%d identifier="%s", storage=%s (master uid=%d)', $oldFileUid, $identifier, $storage, @@ -400,24 +371,21 @@ public function processDuplicate(int $masterFileUid, array $masterMetadataRecord private function getMetadataRecords(int $uid): array { - $metadataQueryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_file_metadata"); - $metadataQueryBuilder->addSelect("*"); - $metadata = $metadataQueryBuilder->from("sys_file_metadata") + $metadataQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_metadata'); + $metadataQueryBuilder->addSelect('*'); + + return $metadataQueryBuilder->from('sys_file_metadata') ->where( $metadataQueryBuilder->expr()->eq( - "file", - $metadataQueryBuilder->createNamedParameter($uid, \PDO::PARAM_INT) + 'file', + $metadataQueryBuilder->createNamedParameter($uid, Connection::PARAM_INT) ) ) - ->executeQuery()->fetchAllAssociative(); - - return $metadata; + ->executeQuery() + ->fetchAllAssociative(); } /** - * @param int $masterFileUid - * @param int $oldFileUid - * @return bool * @throws Exception */ private function findAndUpdateReferences(int $masterFileUid, int $oldFileUid): void @@ -428,8 +396,7 @@ private function findAndUpdateReferences(int $masterFileUid, int $oldFileUid): v return; } - $deleteRecords = true; - $tableRows = null; + $tableRows = []; while ($referenceRow = $referenceStatement->fetchAssociative()) { $tableRows[] = [ $referenceRow['hash'], @@ -445,13 +412,13 @@ private function findAndUpdateReferences(int $masterFileUid, int $oldFileUid): v } } if ($this->output->isVerbose()) { - $this->output->writeln(" -> Updated sys_refindex"); + $this->output->writeln(' -> Updated sys_refindex'); $tableHeaders = [ - "hash", - "tablename", - "recuid", - "field", - "softref_key", + 'hash', + 'tablename', + 'recuid', + 'field', + 'softref_key', ]; $this->output->table($tableHeaders, $tableRows); } @@ -459,29 +426,28 @@ private function findAndUpdateReferences(int $masterFileUid, int $oldFileUid): v public function getSysRefIndexData(int $oldFileUid): Result { - $referenceQueryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_refindex"); + $referenceQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_refindex'); $referenceExpr = $referenceQueryBuilder->expr(); - $referenceStatement = $referenceQueryBuilder->select("*") - ->from("sys_refindex") + return $referenceQueryBuilder->select('*') + ->from('sys_refindex') ->where( $referenceExpr->eq( - "ref_table", - $referenceQueryBuilder->createNamedParameter("sys_file") + 'ref_table', + $referenceQueryBuilder->createNamedParameter('sys_file') ), $referenceExpr->eq( - "ref_uid", + 'ref_uid', $referenceQueryBuilder->createNamedParameter($oldFileUid) ), $referenceExpr->neq( - "tablename", - $referenceQueryBuilder->createNamedParameter("sys_file_metadata") + 'tablename', + $referenceQueryBuilder->createNamedParameter('sys_file_metadata') ) ) ->executeQuery(); - return $referenceStatement; } - private function updateReferencedRecord(int $masterFileUid, array $referenceRow) + private function updateReferencedRecord(int $masterFileUid, array $referenceRow): void { if (empty($referenceRow['softref_key'])) { $value = $masterFileUid; @@ -492,7 +458,7 @@ private function updateReferencedRecord(int $masterFileUid, array $referenceRow) ->from($referenceRow['tablename']) ->where( $recordQueryBuilder->expr()->eq( - "uid", + 'uid', $recordQueryBuilder->createNamedParameter($referenceRow['recuid']) ) ) @@ -503,7 +469,7 @@ private function updateReferencedRecord(int $masterFileUid, array $referenceRow) // update file references $old = 't3://file?uid=' . $referenceRow['ref_uid']; $new = 't3://file?uid=' . $masterFileUid; - $value = preg_replace('/' . preg_quote($old, '/') . '([^\d]|$)' . '/i', $new . '$1', $value); + $value = preg_replace('/' . preg_quote($old, '/') . '([^\d]|$)' . '/i', $new . '$1', (string)$value); // update rte_ckeditor_image references if (ExtensionManagementUtility::isLoaded('rte_ckeditor_image')) { @@ -522,59 +488,52 @@ private function updateReferencedRecord(int $masterFileUid, array $referenceRow) ->where( $recordUpdateExpr->eq( 'uid', - $recordUpdateQueryBuilder->createNamedParameter($referenceRow['recuid'], \PDO::PARAM_INT) + $recordUpdateQueryBuilder->createNamedParameter($referenceRow['recuid'], Connection::PARAM_INT) ) )->executeStatement(); } - private function updateReference(int $masterFileUid, array $referenceRow) + private function updateReference(int $masterFileUid, array $referenceRow): void { $referenceUpdateQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_refindex'); $referenceUpdateExpr = $referenceUpdateQueryBuilder->expr(); $referenceUpdateQueryBuilder->update('sys_refindex') - ->set('ref_uid', $masterFileUid)->where($referenceUpdateExpr->eq( - 'hash', - $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['hash'], \PDO::PARAM_STR) - ), $referenceUpdateExpr->eq( - 'tablename', - $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['tablename'], \PDO::PARAM_STR) - ), $referenceUpdateExpr->eq( - 'recuid', - $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['recuid'], \PDO::PARAM_STR) - ), $referenceUpdateExpr->eq( - 'field', - $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['field'], \PDO::PARAM_STR) - ), $referenceUpdateExpr->eq( - 'ref_table', - $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['ref_table'], \PDO::PARAM_STR) - ), $referenceUpdateExpr->eq( - 'ref_uid', - $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['ref_uid'], \PDO::PARAM_STR) - ))->executeStatement(); + ->set('ref_uid', $masterFileUid)->where( + $referenceUpdateExpr->eq( + 'hash', + $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['hash'], Connection::PARAM_STR) + ), + $referenceUpdateExpr->eq( + 'tablename', + $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['tablename'], Connection::PARAM_STR) + ), + $referenceUpdateExpr->eq( + 'recuid', + $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['recuid'], Connection::PARAM_STR) + ), + $referenceUpdateExpr->eq( + 'field', + $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['field'], Connection::PARAM_STR) + ), + $referenceUpdateExpr->eq( + 'ref_table', + $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['ref_table'], Connection::PARAM_STR) + ), + $referenceUpdateExpr->eq( + 'ref_uid', + $referenceUpdateQueryBuilder->createNamedParameter($referenceRow['ref_uid'], Connection::PARAM_STR) + ) + )->executeStatement(); } - private function deleteOldFileRecord(int $oldFileUid) + private function deleteOldFileRecord(int $oldFileUid): void { $fileDeleteQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file'); $fileDeleteQueryBuilder->delete('sys_file') ->where( $fileDeleteQueryBuilder->expr()->eq( 'uid', - $fileDeleteQueryBuilder->createNamedParameter($oldFileUid, \PDO::PARAM_INT) - ) - ) - ->executeStatement(); - } - - private function markOldFileReferenceRecordDeleted(int $oldFileUid) - { - $fileDeleteQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_reference'); - $fileDeleteQueryBuilder->update('sys_file_reference') - ->set('deleted', 1) - ->where( - $fileDeleteQueryBuilder->expr()->eq( - 'uid_local', - $fileDeleteQueryBuilder->createNamedParameter($oldFileUid, \PDO::PARAM_INT) + $fileDeleteQueryBuilder->createNamedParameter($oldFileUid, Connection::PARAM_INT) ) ) ->executeStatement(); @@ -601,7 +560,7 @@ private function findAndDeleteOldProcessedFile(int $oldFileUid): void ->where( $recordQueryBuilder->expr()->eq( 'original', - $recordQueryBuilder->createNamedParameter($oldFileUid, \PDO::PARAM_INT) + $recordQueryBuilder->createNamedParameter($oldFileUid, Connection::PARAM_INT) ) ) ->executeStatement(); diff --git a/Classes/Metadata/MetadataUpdateHandler.php b/Classes/Metadata/MetadataUpdateHandler.php index b55e740..ab8113c 100644 --- a/Classes/Metadata/MetadataUpdateHandler.php +++ b/Classes/Metadata/MetadataUpdateHandler.php @@ -7,38 +7,35 @@ use Doctrine\DBAL\Exception; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Style\SymfonyStyle; +use TYPO3\CMS\Core\Database\Connection; use TYPO3\CMS\Core\Database\ConnectionPool; class MetadataUpdateHandler { - - /** - * @var String|bool - */ - private $force = false; + private string|bool $force = false; public function __construct( - private bool $dryRun, - private InputInterface $input, - private SymfonyStyle $output, - private array $fieldsToCheck, - private ConnectionPool $connectionPool + private readonly bool $dryRun, + private readonly InputInterface $input, + private readonly SymfonyStyle $output, + private readonly array $fieldsToCheck, + private readonly ConnectionPool $connectionPool ) { - $this->force = $input->getOption("force"); // force will be null if no value is passed -> overwrite + $this->force = $input->getOption('force'); // force will be null if no value is passed -> overwrite if ($this->force === null) { - $this->force = "overwrite"; + $this->force = 'overwrite'; } } - /** - * @param int $masterFileUid - * @param int $oldFileUid - * @return bool * @throws Exception */ - public function updateMetadata(int $masterFileUid, array $masterMetadataRecords, int $oldFileUid, array $oldMetadataRecords): bool - { + public function updateMetadata( + int $masterFileUid, + array $masterMetadataRecords, + int $oldFileUid, + array $oldMetadataRecords + ): bool { $deleteRecords = true; // iterate over all languages @@ -65,17 +62,12 @@ public function updateMetadata(int $masterFileUid, array $masterMetadataRecords, return $deleteRecords; } - - /** - * @param MetadataUpdateObject $metadata - * @return bool - */ public function updateMetadataRecord(MetadataUpdateObject $metadata): bool { if ($metadata->isOldEmtpyt() || $metadata->isOldSameAsMaster()) { // check if record is empty or if the values are the same as in master if ($this->output->isVerbose()) { - $this->output->writeln("\tOld metadata " . $oldUid . " is empty or same as in master for sys_language_uid " . $languageUid . ""); + $this->output->writeln("\tOld metadata " . $metadata->getOldUid() . ' is empty or same as in master for sys_language_uid ' . $metadata->getLanguageUid() . ''); } $this->metadataHandleNoUpdate($metadata->getLanguageUid(), $metadata->getOldUid()); @@ -84,9 +76,9 @@ public function updateMetadataRecord(MetadataUpdateObject $metadata): bool if ($this->output->isVerbose()) { if ($metadata->isMasterEmpty()) { - $this->output->writeln("\tMaster metadata " . $metadata->getMasterUid() . " is empty for sys_language_uid " . $metadata->getLanguageUid() . ""); - } else if ($this->force !== false) { - if ($this->force !== "keep") { + $this->output->writeln("\tMaster metadata " . $metadata->getMasterUid() . ' is empty for sys_language_uid ' . $metadata->getLanguageUid() . ''); + } elseif ($this->force !== false) { + if ($this->force !== 'keep') { $this->output->writeln("\tForce overwriting metadata in master."); } else { $this->output->writeln("\tForce keeping metadata in master."); @@ -105,8 +97,6 @@ public function updateMetadataRecord(MetadataUpdateObject $metadata): bool /** * @param $languageUid - * @param mixed $oldUid - * @return void */ public function metadataHandleNoUpdate($languageUid, mixed $oldUid): void { @@ -120,19 +110,15 @@ public function metadataHandleNoUpdate($languageUid, mixed $oldUid): void } } - /** - * @param MetadataUpdateObject $metadata - * @return void - */ public function metadataHandleUpdate(MetadataUpdateObject $metadata): void { if ($this->force === false || - $this->force === "overwrite" || - $metadata->isMasterEmpty() && $this->force === "keep-nonempty") { + $this->force === 'overwrite' || + $metadata->isMasterEmpty() && $this->force === 'keep-nonempty') { if ($this->output->isVerbose()) { - $this->output->writeln("\t -> " . ($metadata->getMasterUid() ? "Updating" : "Creating") . " master metadata record"); + $this->output->writeln("\t -> " . ($metadata->getMasterUid() ? 'Updating' : 'Creating') . ' master metadata record'); } if (!$this->dryRun) { @@ -145,7 +131,7 @@ public function metadataHandleUpdate(MetadataUpdateObject $metadata): void } if ($this->output->isVerbose()) { - $this->output->writeln("\t -> Deleting old metadata record " . $metadata->getOldUid() . ""); + $this->output->writeln("\t -> Deleting old metadata record " . $metadata->getOldUid() . ''); } if (!$this->dryRun) { @@ -154,18 +140,14 @@ public function metadataHandleUpdate(MetadataUpdateObject $metadata): void } } - /** - * @param MetadataUpdateObject $metadata - * @return false - */ public function metadataHandleConflict(MetadataUpdateObject $metadata): bool { $interactive = $this->input->getOption('interactive'); - $this->output->writeln("\tOld metadata " . $metadata->getOldUid() . " with sys_language_uid " . $metadata->getLanguageUid() . " is not empty and conflicts with the master data. " . ($interactive ? "Please choose what to do." : "Not deleting this record") . "."); + $this->output->writeln("\tOld metadata " . $metadata->getOldUid() . ' with sys_language_uid ' . $metadata->getLanguageUid() . ' is not empty and conflicts with the master data. ' . ($interactive ? 'Please choose what to do.' : 'Not deleting this record') . '.'); if ($this->output->isVerbose()) { - $this->output->writeln("\t -> Old metadata : " . json_encode($metadata->getOldClean()) . ""); - $this->output->writeln("\t -> Master metadata: " . json_encode($metadata->getMasterClean()) . ""); + $this->output->writeln("\t -> Old metadata : " . json_encode($metadata->getOldClean()) . ''); + $this->output->writeln("\t -> Master metadata: " . json_encode($metadata->getMasterClean()) . ''); } if ($interactive) { @@ -180,7 +162,7 @@ public function metadataHandleConflict(MetadataUpdateObject $metadata): bool $this->metadataHandleNoUpdate($metadata->getLanguageUid(), $metadata->getOldUid()); return true; case 's': - $this->output->writeln("\tSkipping record. Not deleting any duplicate records related to file " . $metadata->getMasterFileUid() . ""); + $this->output->writeln("\tSkipping record. Not deleting any duplicate records related to file " . $metadata->getMasterFileUid() . ''); return false; case 'h': default: @@ -198,11 +180,10 @@ public function metadataHandleConflict(MetadataUpdateObject $metadata): bool return false; } - - private function updateMasterMetadata(int $masterMetadataUid, array $metadata) + private function updateMasterMetadata(int $masterMetadataUid, array $metadata): void { - $metadataUpdateQueryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_file_metadata"); - $metadataUpdateQueryBuilder->update("sys_file_metadata"); + $metadataUpdateQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_metadata'); + $metadataUpdateQueryBuilder->update('sys_file_metadata'); foreach ($this->fieldsToCheck as $field) { if (!isset($metadata[$field])) { $this->output->writeln("\t\tField \'" . $field . "\' does not exist"); @@ -213,54 +194,50 @@ private function updateMasterMetadata(int $masterMetadataUid, array $metadata) $metadataUpdateQueryBuilder->where( $metadataUpdateQueryBuilder->expr()->eq( 'uid', - $metadataUpdateQueryBuilder->createNamedParameter($masterMetadataUid, \PDO::PARAM_INT) + $metadataUpdateQueryBuilder->createNamedParameter($masterMetadataUid, Connection::PARAM_INT) ) ) ->executeStatement(); } - private function createMasterMetadata(int $masterFileUid, array $metadata) + private function createMasterMetadata(int $masterFileUid, array $metadata): void { unset($metadata['uid']); $metadata['file'] = $masterFileUid; - $metadataUpdateQueryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_file_metadata"); - $metadataUpdateQueryBuilder->insert("sys_file_metadata") + $metadataUpdateQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_metadata'); + $metadataUpdateQueryBuilder->insert('sys_file_metadata') ->values($metadata) ->executeStatement(); } - /** - * @param int $uid - * @return mixed - */ - public function deleteMetadataRecord(int $uid) + public function deleteMetadataRecord(int $uid): int { - $queryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_file_metadata"); + $queryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_file_metadata'); return $queryBuilder - ->delete("sys_file_metadata") + ->delete('sys_file_metadata') ->where( $queryBuilder->expr()->eq( - "uid", - $queryBuilder->createNamedParameter($uid, \PDO::PARAM_INT) + 'uid', + $queryBuilder->createNamedParameter($uid, Connection::PARAM_INT) ) ) ->executeStatement(); } - private function deleteMetadataReference(int $uid) + private function deleteMetadataReference(int $uid): void { - $referenceDeleteQueryBuilder = $this->connectionPool->getQueryBuilderForTable("sys_refindex"); + $referenceDeleteQueryBuilder = $this->connectionPool->getQueryBuilderForTable('sys_refindex'); $referenceDeleteExpr = $referenceDeleteQueryBuilder->expr(); $referenceDeleteQueryBuilder - ->delete("sys_refindex") + ->delete('sys_refindex') ->where( $referenceDeleteExpr->eq( - "tablename", - $referenceDeleteQueryBuilder->createNamedParameter("sys_file_metadata", \PDO::PARAM_STR) + 'tablename', + $referenceDeleteQueryBuilder->createNamedParameter('sys_file_metadata', Connection::PARAM_STR) ), $referenceDeleteExpr->eq( - "recuid", - $referenceDeleteQueryBuilder->createNamedParameter($uid, \PDO::PARAM_INT) + 'recuid', + $referenceDeleteQueryBuilder->createNamedParameter($uid, Connection::PARAM_INT) ) )->executeStatement(); } diff --git a/Classes/Metadata/MetadataUpdateObject.php b/Classes/Metadata/MetadataUpdateObject.php index 45ac764..01e5570 100644 --- a/Classes/Metadata/MetadataUpdateObject.php +++ b/Classes/Metadata/MetadataUpdateObject.php @@ -6,14 +6,12 @@ class MetadataUpdateObject { - public function __construct( public readonly int $masterFileUid, public readonly array $masterMetadata, public readonly array $oldMetadata, public readonly array $fieldsToCheck - ) { - } + ) {} public function getOldMetadata(): array { diff --git a/Tests/Functional/Command/UnduplicateCommandTest.php b/Tests/Functional/Command/UnduplicateCommandTest.php index e0e250e..4a7e598 100644 --- a/Tests/Functional/Command/UnduplicateCommandTest.php +++ b/Tests/Functional/Command/UnduplicateCommandTest.php @@ -5,13 +5,11 @@ namespace ElementareTeilchen\Unduplicator\Tests\Functional\Command; use PHPUnit\Framework\Attributes\Test; -use RuntimeException; use TYPO3\TestingFramework\Core\Functional\FunctionalTestCase; class UnduplicateCommandTest extends FunctionalTestCase { - - const BASE_COMMAND = 'unduplicate:sysfile -n'; + public const BASE_COMMAND = 'unduplicate:sysfile -n'; #[Test] public function unduplicateCommandReturnsZeroIfNoDuplicates(): void { @@ -88,7 +86,6 @@ class UnduplicateCommandTest extends FunctionalTestCase self::assertEquals(0, $result['status']); } - #[Test] public function unduplicateCommandFixesDuplicatesWithMetadata(): void { $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata.csv'); @@ -100,7 +97,6 @@ class UnduplicateCommandTest extends FunctionalTestCase self::assertEquals(0, $result['status']); } - #[Test] public function unduplicateCommandFixesDuplicatesWithLanguagesMetadata(): void { $this->importCSVDataSet(__DIR__ . '/DataSet/sys_file_duplicates_with_metadata_languages.csv'); @@ -175,7 +171,6 @@ class UnduplicateCommandTest extends FunctionalTestCase self::assertEquals(0, $result['status']); } - /** * based on TYPO3\CMS\Core\Tests\Functional\Command\AbstractCommandTest::executeConsoleCommand * we had to change path for typo3 command because EXT:core/bin/typo3 does not exist in Composer installation @@ -184,8 +179,9 @@ protected function executeConsoleCommand(string $cmdline, ...$args): array { $typo3File = __DIR__ . '/../../../bin/typo3'; if (!file_exists($typo3File)) { - throw new RuntimeException( - sprintf('Executable file not found (using path <%s>). Make sure config:bin-dir is set to \'bin\' in composer.json', $typo3File) + throw new \RuntimeException( + sprintf('Executable file not found (using path <%s>). Make sure config:bin-dir is set to \'bin\' in composer.json', $typo3File), + 2791340354 ); } From 52f401f96e12ba7508d52024379349568f466af0 Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Thu, 18 Dec 2025 17:32:11 +0100 Subject: [PATCH 6/8] Fix issues preventing the functional tests from succeeding --- Classes/Command/UnduplicateCommand.php | 8 +++++--- Classes/Metadata/MetadataUpdateHandler.php | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Classes/Command/UnduplicateCommand.php b/Classes/Command/UnduplicateCommand.php index 6a96ef2..09e648f 100644 --- a/Classes/Command/UnduplicateCommand.php +++ b/Classes/Command/UnduplicateCommand.php @@ -317,12 +317,14 @@ private function findDuplicateFilesForIdentifier(string $identifier, int $storag $fileQueryBuilder->expr()->eq( 'storage', $fileQueryBuilder->createNamedParameter($storage, Connection::PARAM_INT) + ), + $fileQueryBuilder->expr()->comparison( + 'MD5(' . $fileQueryBuilder->quoteIdentifier('identifier') . ')', + $fileQueryBuilder->expr()::EQ, + 'MD5(' . $fileQueryBuilder->createNamedParameter($identifier, Connection::PARAM_STR) . ')' ) )->orderBy('uid', $this->keepOldest ? 'ASC' : 'DESC'); - $whereClause = 'MD5(identifier) = MD5(' . $fileQueryBuilder->createNamedParameter($identifier, Connection::PARAM_STR) . ')'; - $fileQueryBuilder->add('where', $whereClause); - return $fileQueryBuilder->executeQuery() ->fetchAllAssociative(); } diff --git a/Classes/Metadata/MetadataUpdateHandler.php b/Classes/Metadata/MetadataUpdateHandler.php index ab8113c..f5edd58 100644 --- a/Classes/Metadata/MetadataUpdateHandler.php +++ b/Classes/Metadata/MetadataUpdateHandler.php @@ -12,7 +12,7 @@ class MetadataUpdateHandler { - private string|bool $force = false; + private null|string|bool $force = false; public function __construct( private readonly bool $dryRun, From d673b9b99882f8a92481a394f2b5301293ec21f9 Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Mon, 16 Feb 2026 13:58:43 +0100 Subject: [PATCH 7/8] Increase timeout for container test in test script --- Build/Scripts/runTests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Build/Scripts/runTests.sh b/Build/Scripts/runTests.sh index befc2b6..4629ab4 100755 --- a/Build/Scripts/runTests.sh +++ b/Build/Scripts/runTests.sh @@ -20,7 +20,8 @@ waitFor() { echo \"Can not connect to ${HOST} port ${PORT}. Aborting.\"; exit 1; fi; - sleep 1; + # Increase timeout in case a container needs more than 10 seconds to boot up + sleep 3; COUNT=\$((COUNT + 1)); done; " From 725c449dfc00b2e65c6e827e02734fac2980b6de Mon Sep 17 00:00:00 2001 From: Richard Klein Date: Mon, 16 Feb 2026 15:01:31 +0100 Subject: [PATCH 8/8] Revert some changes in github workflow due to failing tests --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6af326b..b2aa79e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: all_core_13: name: "all core-13" - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 strategy: # This prevents cancellation of matrix job runs, if one/two already failed and let the # rest of the matrix jobs be executed anyway. @@ -25,7 +25,7 @@ jobs: minMax: [ 'composerInstall'] steps: - name: "Checkout" - uses: actions/checkout@v6 + uses: actions/checkout@v4 - name: "Composer" run: Build/Scripts/runTests.sh -p ${{ matrix.php }} -s ${{ matrix.minMax }}