diff --git a/.github/workflows/integration-partitioning.yml b/.github/workflows/integration-partitioning.yml new file mode 100644 index 0000000000000..cae106152875f --- /dev/null +++ b/.github/workflows/integration-partitioning.yml @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: MIT +name: Integration mariadb-partitioning + +on: + pull_request: + +permissions: + contents: read + +concurrency: + group: integration-mariadb-partitioning-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest-low + + outputs: + src: ${{ steps.changes.outputs.src}} + + steps: + - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/*.php' + - '**/lib/**' + - '**/tests/**' + - '**/vendor-bin/**' + - 'build/integration/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - 'core/shipped.json' + + integration-mariadb-partitioning: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + fail-fast: false + matrix: + test-suite: + - 'capabilities_features' + - 'collaboration_features' + - 'comments_features' + - '--tags ~@requires-s3 dav_features' + - 'features' + - 'federation_features' + - '--tags ~@large files_features' + - 'filesdrop_features' + - 'file_conversions' + - 'files_reminders' + - 'openldap_features' + - 'openldap_numerical_features' + - 'ldap_features' + - 'routing_features' + - 'setup_features' + - 'sharees_features' + - 'sharing_features' + - 'theming_features' + - 'videoverification_features' + - 'guests_features' + + php-versions: ['8.5'] + mysql-versions: ['9.7'] + guests-versions: ['main'] + spreed-versions: ['main'] + activity-versions: ['master'] + + services: + redis: + image: ghcr.io/nextcloud/continuous-integration-redis:latest # zizmor: ignore[unpinned-images] + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + ports: + - 6379:6379/tcp + openldap: + image: ghcr.io/nextcloud/continuous-integration-openldap:openldap-8 # zizmor: ignore[unpinned-images] + ports: + - 389:389 + - 636:636 + env: + SLAPD_DOMAIN: nextcloud.ci + SLAPD_ORGANIZATION: Nextcloud + SLAPD_PASSWORD: admin + SLAPD_ADDITIONAL_MODULES: memberof + + mysql: + image: mysql:${{ matrix.mysql-versions }} # zizmor: ignore[unpinned-images] + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: oc_autotest + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + + steps: + - name: Checkout server + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + submodules: true + + - name: Checkout Talk app + if: ${{ matrix.test-suite == 'videoverification_features' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: nextcloud/spreed + path: apps/spreed + ref: ${{ matrix.spreed-versions }} + + - name: Checkout Guests app + if: ${{ matrix.test-suite == 'guests_features' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: nextcloud/guests + path: apps/guests + ref: ${{ matrix.guests-versions }} + + - name: Checkout Activity app + if: ${{ matrix.test-suite == 'sharing_features' }} + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: nextcloud/activity + path: apps/activity + ref: ${{ matrix.activity-versions }} + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 #v2.37.2 + timeout-minutes: 5 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, imagick, intl, json, ldap, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql + coverage: none + ini-file: development + ini-values: disable_functions="" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: | + composer install + + - name: Enable ONLY_FULL_GROUP_BY MySQL option + run: | + echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + + - name: Set up Talk dependencies + if: ${{ matrix.test-suite == 'videoverification_features' }} + working-directory: apps/spreed + run: composer i --no-dev + + - name: Set up Nextcloud + env: + DB_PORT: 4444 + run: | + mkdir data + ./occ maintenance:install --verbose ${{ contains(matrix.test-suite,'ldap') && '--data-dir=/dev/shm/nc_int' || '' }} --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + ./occ config:system:set hashing_default_password --value=true --type=boolean + + - name: Enable sharding for oc_filecache + run: | + echo "alter table oc_filecache drop primary key, add primary key (fileid, storage);" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + echo "alter table oc_filecache partition by hash(storage) partitions 5;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + + - name: Configure caching + if: ${{ contains(matrix.test-suite,'ldap') }} + run: | + ./occ config:system:set redis host --value=localhost + ./occ config:system:set redis port --value=6379 --type=integer + ./occ config:system:set redis timeout --value=0 --type=integer + ./occ config:system:set memcache.local --value='\OC\Memcache\Redis' + ./occ config:system:set memcache.distributed --value='\OC\Memcache\Redis' + + - name: Run integration + working-directory: build/integration + env: + LDAP_HOST: localhost + run: bash run.sh ${{ matrix.test-suite }} no-tail-log + + - name: Show partition info + run: | + echo "SELECT table_name, partition_name, table_rows FROM information_schema.partitions WHERE partition_name is not null;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + echo "SELECT table_rows FROM information_schema.tables WHERE table_name = 'oc_filecache';" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + + - name: Print logs + if: always() + run: | + cat $(./occ log:file |grep "Log file"|cut -d" " -f3) + docker ps -a + docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done + + summary: + permissions: + contents: none + runs-on: ubuntu-latest-low + needs: [changes, integration-mariadb-partitioning] + + if: always() + + name: integration-mariadb-partitioning-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.integration-mariadb-partitioning.result != 'success' }}; then exit 1; fi diff --git a/.github/workflows/phpunit-mysql-partitioning.yml b/.github/workflows/phpunit-mysql-partitioning.yml new file mode 100644 index 0000000000000..b07ed6bfdd3f5 --- /dev/null +++ b/.github/workflows/phpunit-mysql-partitioning.yml @@ -0,0 +1,158 @@ +# This workflow is provided via the organization template repository +# +# https://github.com/nextcloud/.github +# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization +# +# SPDX-FileCopyrightText: 2022-2024 Nextcloud GmbH and Nextcloud contributors +# SPDX-License-Identifier: MIT + +name: PHPUnit mysql partitioning + +on: + pull_request: + schedule: + - cron: "5 2 * * *" + +permissions: + contents: read + +concurrency: + group: phpunit-mysql-partitioning-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + changes: + runs-on: ubuntu-latest-low + permissions: + contents: read + pull-requests: read + + outputs: + src: ${{ steps.changes.outputs.src }} + + steps: + - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2 + id: changes + continue-on-error: true + with: + filters: | + src: + - '.github/workflows/**' + - '3rdparty/**' + - '**/appinfo/**' + - '**/lib/**' + - '**/templates/**' + - '**/tests/**' + - 'vendor/**' + - 'vendor-bin/**' + - '.php-cs-fixer.dist.php' + - 'composer.json' + - 'composer.lock' + - '**.php' + + phpunit-mysql-partitioning: + runs-on: ubuntu-latest + + needs: changes + if: needs.changes.outputs.src != 'false' + + strategy: + fail-fast: false + matrix: + include: + - mysql-versions: '9.7' + php-versions: '8.5' + + name: MySQL ${{ matrix.mysql-versions }} (PHP ${{ matrix.php-versions }}) - database tests + + services: + cache: + image: ghcr.io/nextcloud/continuous-integration-redis:latest # zizmor: ignore[unpinned-images] + ports: + - 6379:6379/tcp + options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3 + + mysql: + image: mysql:${{ matrix.mysql-versions }} # zizmor: ignore[unpinned-images] + ports: + - 4444:3306/tcp + env: + MYSQL_ROOT_PASSWORD: rootpassword + MYSQL_USER: oc_autotest + MYSQL_PASSWORD: nextcloud + MYSQL_DATABASE: oc_autotest + options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10 + + steps: + - name: Checkout server + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + submodules: true + + - name: Set up php ${{ matrix.php-versions }} + uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2.37.2 + timeout-minutes: 5 + with: + php-version: ${{ matrix.php-versions }} + # https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation + extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql + coverage: none + ini-file: development + ini-values: disable_functions="" + env: + fail-fast: true + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up dependencies + run: composer i + + - name: Enable ONLY_FULL_GROUP_BY MySQL option + run: | + echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword + + - name: Set up Nextcloud + env: + DB_PORT: 4444 + run: | + mkdir data + cp tests/redis.config.php config/ + cp tests/preseed-config.php config/config.php + ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin + php -f tests/enable_all.php + + - name: Enable sharding for oc_filecache + run: | + echo "alter table oc_filecache drop primary key, add primary key (fileid, storage);" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + echo "alter table oc_filecache partition by hash(storage) partitions 5;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + + - name: PHPUnit + run: composer run test:db -- --log-junit junit.xml + env: + DB_ROOT_USER: root + DB_ROOT_PASS: rootpassword + + - name: Show partition info + run: | + echo "SELECT table_name, partition_name, table_rows FROM information_schema.partitions WHERE partition_name is not null;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + echo "SELECT table_rows FROM information_schema.tables WHERE table_name = 'oc_filecache';" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud + + - name: Print logs + if: always() + run: | + cat data/nextcloud.log + + summary: + permissions: + contents: none + runs-on: ubuntu-latest-low + needs: [changes, phpunit-mysql-partitioning] + + if: always() + + name: phpunit-mysql-partitioning-summary + + steps: + - name: Summary status + run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-mysql-partitioning.result != 'success' }}; then exit 1; fi