Skip to content

test: add ci test for mariadb with filecache partitioning #62491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
icewind1991 wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

test: add ci test for mariadb with filecache partitioning #62491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d34c992
test: add ci test for mariadb with filecache partitioning
icewind1991 Jul 24, 2026
ae714da
add ci for integration tests with mysql partitioning
icewind1991 Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
219 changes: 219 additions & 0 deletions .github/workflows/integration-partitioning.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
# SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
# SPDX-License-Identifier: MIT
name: Integration mariadb-partitioning

on:
pull_request:

permissions:
contents: read

concurrency:
group: integration-mariadb-partitioning-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
changes:
runs-on: ubuntu-latest-low

outputs:
src: ${{ steps.changes.outputs.src}}

steps:
- uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3
id: changes
continue-on-error: true
with:
filters: |
src:
- '.github/workflows/**'
- '3rdparty/**'
- '**/*.php'
- '**/lib/**'
- '**/tests/**'
- '**/vendor-bin/**'
- 'build/integration/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
- 'core/shipped.json'

integration-mariadb-partitioning:
runs-on: ubuntu-latest

needs: changes
if: needs.changes.outputs.src != 'false'

strategy:
fail-fast: false
matrix:
test-suite:
- 'capabilities_features'
- 'collaboration_features'
- 'comments_features'
- '--tags ~@requires-s3 dav_features'
- 'features'
- 'federation_features'
- '--tags ~@large files_features'
- 'filesdrop_features'
- 'file_conversions'
- 'files_reminders'
- 'openldap_features'
- 'openldap_numerical_features'
- 'ldap_features'
- 'routing_features'
- 'setup_features'
- 'sharees_features'
- 'sharing_features'
- 'theming_features'
- 'videoverification_features'
- 'guests_features'

php-versions: ['8.5']
mysql-versions: ['9.7']
guests-versions: ['main']
spreed-versions: ['main']
activity-versions: ['master']

services:
redis:
image: ghcr.io/nextcloud/continuous-integration-redis:latest # zizmor: ignore[unpinned-images]
options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3
ports:
- 6379:6379/tcp
openldap:
image: ghcr.io/nextcloud/continuous-integration-openldap:openldap-8 # zizmor: ignore[unpinned-images]
ports:
- 389:389
- 636:636
env:
SLAPD_DOMAIN: nextcloud.ci
SLAPD_ORGANIZATION: Nextcloud
SLAPD_PASSWORD: admin
SLAPD_ADDITIONAL_MODULES: memberof

mysql:
image: mysql:${{ matrix.mysql-versions }} # zizmor: ignore[unpinned-images]
ports:
- 4444:3306/tcp
env:
MYSQL_ROOT_PASSWORD: rootpassword
MYSQL_USER: oc_autotest
MYSQL_PASSWORD: nextcloud
MYSQL_DATABASE: oc_autotest
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10

steps:
- name: Checkout server
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
submodules: true

- name: Checkout Talk app
if: ${{ matrix.test-suite == 'videoverification_features' }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
repository: nextcloud/spreed
path: apps/spreed
ref: ${{ matrix.spreed-versions }}

- name: Checkout Guests app
if: ${{ matrix.test-suite == 'guests_features' }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
repository: nextcloud/guests
path: apps/guests
ref: ${{ matrix.guests-versions }}

- name: Checkout Activity app
if: ${{ matrix.test-suite == 'sharing_features' }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
repository: nextcloud/activity
path: apps/activity
ref: ${{ matrix.activity-versions }}

- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 #v2.37.2
timeout-minutes: 5
with:
php-version: ${{ matrix.php-versions }}
# https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, imagick, intl, json, ldap, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql
coverage: none
ini-file: development
ini-values: disable_functions=""
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Set up dependencies
run: |
composer install

- name: Enable ONLY_FULL_GROUP_BY MySQL option
run: |
echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword
echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword

- name: Set up Talk dependencies
if: ${{ matrix.test-suite == 'videoverification_features' }}
working-directory: apps/spreed
run: composer i --no-dev

- name: Set up Nextcloud
env:
DB_PORT: 4444
run: |
mkdir data
./occ maintenance:install --verbose ${{ contains(matrix.test-suite,'ldap') && '--data-dir=/dev/shm/nc_int' || '' }} --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
./occ config:system:set hashing_default_password --value=true --type=boolean

- name: Enable sharding for oc_filecache
run: |
echo "alter table oc_filecache drop primary key, add primary key (fileid, storage);" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud
echo "alter table oc_filecache partition by hash(storage) partitions 5;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud

- name: Configure caching
if: ${{ contains(matrix.test-suite,'ldap') }}
run: |
./occ config:system:set redis host --value=localhost
./occ config:system:set redis port --value=6379 --type=integer
./occ config:system:set redis timeout --value=0 --type=integer
./occ config:system:set memcache.local --value='\OC\Memcache\Redis'
./occ config:system:set memcache.distributed --value='\OC\Memcache\Redis'

- name: Run integration
working-directory: build/integration
env:
LDAP_HOST: localhost
run: bash run.sh ${{ matrix.test-suite }} no-tail-log

- name: Show partition info
run: |
echo "SELECT table_name, partition_name, table_rows FROM information_schema.partitions WHERE partition_name is not null;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud
echo "SELECT table_rows FROM information_schema.tables WHERE table_name = 'oc_filecache';" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud

- name: Print logs
if: always()
run: |
cat $(./occ log:file |grep "Log file"|cut -d" " -f3)
docker ps -a
docker ps -aq | while read container ; do IMAGE=$(docker inspect --format='{{.Config.Image}}' $container); echo $IMAGE; docker logs $container; echo "\n\n" ; done

summary:
permissions:
contents: none
runs-on: ubuntu-latest-low
needs: [changes, integration-mariadb-partitioning]

if: always()

name: integration-mariadb-partitioning-summary

steps:
- name: Summary status
run: if ${{ needs.changes.outputs.src != 'false' && needs.integration-mariadb-partitioning.result != 'success' }}; then exit 1; fi
158 changes: 158 additions & 0 deletions .github/workflows/phpunit-mysql-partitioning.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
# This workflow is provided via the organization template repository
#
# https://github.com/nextcloud/.github
# https://docs.github.com/en/actions/learn-github-actions/sharing-workflows-with-your-organization
#
# SPDX-FileCopyrightText: 2022-2024 Nextcloud GmbH and Nextcloud contributors
# SPDX-License-Identifier: MIT

name: PHPUnit mysql partitioning

on:
pull_request:
schedule:
- cron: "5 2 * * *"

permissions:
contents: read

concurrency:
group: phpunit-mysql-partitioning-${{ github.head_ref || github.run_id }}
cancel-in-progress: true

jobs:
changes:
runs-on: ubuntu-latest-low
permissions:
contents: read
pull-requests: read

outputs:
src: ${{ steps.changes.outputs.src }}

steps:
- uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
id: changes
continue-on-error: true
with:
filters: |
src:
- '.github/workflows/**'
- '3rdparty/**'
- '**/appinfo/**'
- '**/lib/**'
- '**/templates/**'
- '**/tests/**'
- 'vendor/**'
- 'vendor-bin/**'
- '.php-cs-fixer.dist.php'
- 'composer.json'
- 'composer.lock'
- '**.php'

phpunit-mysql-partitioning:
runs-on: ubuntu-latest

needs: changes
if: needs.changes.outputs.src != 'false'

strategy:
fail-fast: false
matrix:
include:
- mysql-versions: '9.7'
php-versions: '8.5'

name: MySQL ${{ matrix.mysql-versions }} (PHP ${{ matrix.php-versions }}) - database tests

services:
cache:
image: ghcr.io/nextcloud/continuous-integration-redis:latest # zizmor: ignore[unpinned-images]
ports:
- 6379:6379/tcp
options: --health-cmd="redis-cli ping" --health-interval=10s --health-timeout=5s --health-retries=3

mysql:
image: mysql:${{ matrix.mysql-versions }} # zizmor: ignore[unpinned-images]
ports:
- 4444:3306/tcp
env:
MYSQL_ROOT_PASSWORD: rootpassword
MYSQL_USER: oc_autotest
MYSQL_PASSWORD: nextcloud
MYSQL_DATABASE: oc_autotest
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 10

steps:
- name: Checkout server
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
submodules: true

- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2.37.2
timeout-minutes: 5
with:
php-version: ${{ matrix.php-versions }}
# https://docs.nextcloud.com/server/stable/admin_manual/installation/source_installation.html#prerequisites-for-manual-installation
extensions: bz2, ctype, curl, dom, fileinfo, gd, iconv, intl, json, libxml, mbstring, openssl, pcntl, posix, redis, session, simplexml, xmlreader, xmlwriter, zip, zlib, mysql, pdo_mysql
coverage: none
ini-file: development
ini-values: disable_functions=""
env:
fail-fast: true
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Set up dependencies
run: composer i

- name: Enable ONLY_FULL_GROUP_BY MySQL option
run: |
echo "SET GLOBAL sql_mode=(SELECT CONCAT(@@sql_mode,',ONLY_FULL_GROUP_BY'));" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword
echo "SELECT @@sql_mode;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword

- name: Set up Nextcloud
env:
DB_PORT: 4444
run: |
mkdir data
cp tests/redis.config.php config/
cp tests/preseed-config.php config/config.php
./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
php -f tests/enable_all.php

- name: Enable sharding for oc_filecache
run: |
echo "alter table oc_filecache drop primary key, add primary key (fileid, storage);" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud
echo "alter table oc_filecache partition by hash(storage) partitions 5;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud

- name: PHPUnit
run: composer run test:db -- --log-junit junit.xml
env:
DB_ROOT_USER: root
DB_ROOT_PASS: rootpassword

- name: Show partition info
run: |
echo "SELECT table_name, partition_name, table_rows FROM information_schema.partitions WHERE partition_name is not null;" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud
echo "SELECT table_rows FROM information_schema.tables WHERE table_name = 'oc_filecache';" | mysql -h 127.0.0.1 -P 4444 -u root -prootpassword nextcloud

- name: Print logs
if: always()
run: |
cat data/nextcloud.log

summary:
permissions:
contents: none
runs-on: ubuntu-latest-low
needs: [changes, phpunit-mysql-partitioning]

if: always()

name: phpunit-mysql-partitioning-summary

steps:
- name: Summary status
run: if ${{ needs.changes.outputs.src != 'false' && needs.phpunit-mysql-partitioning.result != 'success' }}; then exit 1; fi
Loading