From 91de71cbfe718c97dfcfbcfbf0a857ac3a1a99c7 Mon Sep 17 00:00:00 2001 From: Vitaliy Mogilevskiy Date: Wed, 9 Mar 2022 09:54:19 -0800 Subject: [PATCH] adds 467B942D3A79BD29 key to CI tests Signed-off-by: Vitaliy Mogilevskiy --- .github/workflows/cluster_endtoend_11.yml | 8 +- .github/workflows/cluster_endtoend_12.yml | 8 +- .github/workflows/cluster_endtoend_13.yml | 8 +- .github/workflows/cluster_endtoend_14.yml | 8 +- .github/workflows/cluster_endtoend_15.yml | 8 +- .github/workflows/cluster_endtoend_16.yml | 8 +- .github/workflows/cluster_endtoend_17.yml | 8 +- .github/workflows/cluster_endtoend_18.yml | 8 +- .github/workflows/cluster_endtoend_19.yml | 8 +- .github/workflows/cluster_endtoend_20.yml | 8 +- .github/workflows/cluster_endtoend_21.yml | 8 +- .github/workflows/cluster_endtoend_22.yml | 8 +- .github/workflows/cluster_endtoend_23.yml | 8 +- .github/workflows/cluster_endtoend_24.yml | 8 +- .github/workflows/cluster_endtoend_26.yml | 8 +- .../workflows/cluster_endtoend_mysql80.yml | 13 +- ...cluster_endtoend_onlineddl_declarative.yml | 8 +- .../cluster_endtoend_onlineddl_ghost.yml | 8 +- .../cluster_endtoend_onlineddl_revert.yml | 8 +- .../cluster_endtoend_onlineddl_singleton.yml | 8 +- .../cluster_endtoend_onlineddl_vrepl.yml | 8 +- ...luster_endtoend_onlineddl_vrepl_stress.yml | 8 +- ...cluster_endtoend_onlineddl_vrepl_suite.yml | 8 +- .../workflows/cluster_endtoend_resharding.yml | 8 +- .../cluster_endtoend_resharding_bytes.yml | 8 +- ...cluster_endtoend_tabletmanager_tablegc.yml | 8 +- ...uster_endtoend_tabletmanager_throttler.yml | 8 +- ..._tabletmanager_throttler_custom_config.yml | 8 +- .../workflows/cluster_endtoend_upgrade.yml | 4 +- .../cluster_endtoend_vreplication_basic.yml | 8 +- ...luster_endtoend_vreplication_cellalias.yml | 8 +- .../cluster_endtoend_vreplication_migrate.yml | 8 +- ...luster_endtoend_vreplication_multicell.yml | 8 +- .../cluster_endtoend_vreplication_v2.yml | 8 +- .../cluster_endtoend_vtgate_buffer.yml | 8 +- .../cluster_endtoend_vtgate_concurrentdml.yml | 8 +- .../cluster_endtoend_vtgate_gen4.yml | 8 +- ...cluster_endtoend_vtgate_readafterwrite.yml | 8 +- .../cluster_endtoend_vtgate_reservedconn.yml | 8 +- .../cluster_endtoend_vtgate_schema.yml | 8 +- .../cluster_endtoend_vtgate_topo.yml | 8 +- .../cluster_endtoend_vtgate_transaction.yml | 8 +- .../cluster_endtoend_vtgate_unsharded.yml | 8 +- .../cluster_endtoend_vtgate_vindex.yml | 8 +- .../cluster_endtoend_vtgate_vschema.yml | 8 +- .github/workflows/cluster_endtoend_vtorc.yml | 8 +- .../cluster_endtoend_xb_recovery.yml | 8 +- .github/workflows/e2e_race.yml | 1 + .github/workflows/legacy_local_example.yml | 1 + .github/workflows/local_example.yml | 1 + .github/workflows/region_example.yml | 1 + .github/workflows/unit_test_mysql80.yml | 3 + Makefile | 4 +- docker/bootstrap/Dockerfile.mysql56 | 1 + docker/bootstrap/Dockerfile.mysql57 | 1 + docker/bootstrap/Dockerfile.mysql80 | 1 + docker/lite/install_dependencies.sh | 1 + go.mod | 3 +- go.sum | 12 +- go/test/endtoend/vtgate/system_schema_test.go | 2 +- go/vt/sqlparser/sql.y | 2 +- go/vt/vtctld/vtctld.go | 4 +- go/vt/vtctld/workflow.go | 4 +- go/vt/vtgate/planbuilder/plan_test.go | 25 +-- .../testdata/bypass_keyrange_cases.txt | 174 ------------------ .../planbuilder/testdata/filter_cases.txt | 22 ++- .../planbuilder/testdata/from_cases.txt | 138 ++++---------- .../planbuilder/testdata/select_cases.txt | 2 +- .../testdata/unsupported_cases.txt | 17 +- .../planbuilder/testdata/exec_cases.txt | 14 -- go/vt/workflow/long_polling.go | 58 ++++-- go/vt/workflow/long_polling_test.go | 86 +++++++++ go/vt/workflow/manager.go | 10 +- test/templates/cluster_endtoend_test.tpl | 17 +- test/templates/unit_test.tpl | 2 + 75 files changed, 593 insertions(+), 391 deletions(-) delete mode 100644 go/vt/vtgate/planbuilder/testdata/bypass_keyrange_cases.txt diff --git a/.github/workflows/cluster_endtoend_11.yml b/.github/workflows/cluster_endtoend_11.yml index 4b1b7c2559c..fa32a68046f 100644 --- a/.github/workflows/cluster_endtoend_11.yml +++ b/.github/workflows/cluster_endtoend_11.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml index b28b03b45e1..b8f5af14b5d 100644 --- a/.github/workflows/cluster_endtoend_12.yml +++ b/.github/workflows/cluster_endtoend_12.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_13.yml b/.github/workflows/cluster_endtoend_13.yml index 5ff94e79f10..90f4fc93d3e 100644 --- a/.github/workflows/cluster_endtoend_13.yml +++ b/.github/workflows/cluster_endtoend_13.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_14.yml b/.github/workflows/cluster_endtoend_14.yml index 455540c3347..92d23d650b9 100644 --- a/.github/workflows/cluster_endtoend_14.yml +++ b/.github/workflows/cluster_endtoend_14.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_15.yml b/.github/workflows/cluster_endtoend_15.yml index 62ce1d475c0..554a449a35d 100644 --- a/.github/workflows/cluster_endtoend_15.yml +++ b/.github/workflows/cluster_endtoend_15.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_16.yml b/.github/workflows/cluster_endtoend_16.yml index ee9b34d7ee2..4ceba28b06c 100644 --- a/.github/workflows/cluster_endtoend_16.yml +++ b/.github/workflows/cluster_endtoend_16.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_17.yml b/.github/workflows/cluster_endtoend_17.yml index 7412da90628..7478c0ad22a 100644 --- a/.github/workflows/cluster_endtoend_17.yml +++ b/.github/workflows/cluster_endtoend_17.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml index 04384af9a57..91483420c90 100644 --- a/.github/workflows/cluster_endtoend_18.yml +++ b/.github/workflows/cluster_endtoend_18.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_19.yml b/.github/workflows/cluster_endtoend_19.yml index 31c708e6a93..2fd606fd56f 100644 --- a/.github/workflows/cluster_endtoend_19.yml +++ b/.github/workflows/cluster_endtoend_19.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_20.yml b/.github/workflows/cluster_endtoend_20.yml index a35a7ee69fa..50c8886e17c 100644 --- a/.github/workflows/cluster_endtoend_20.yml +++ b/.github/workflows/cluster_endtoend_20.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_21.yml b/.github/workflows/cluster_endtoend_21.yml index 04b385de829..14294718df1 100644 --- a/.github/workflows/cluster_endtoend_21.yml +++ b/.github/workflows/cluster_endtoend_21.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_22.yml b/.github/workflows/cluster_endtoend_22.yml index 341da50cac6..9d76c03a6da 100644 --- a/.github/workflows/cluster_endtoend_22.yml +++ b/.github/workflows/cluster_endtoend_22.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_23.yml b/.github/workflows/cluster_endtoend_23.yml index a2910ee2f2a..54d72b9699f 100644 --- a/.github/workflows/cluster_endtoend_23.yml +++ b/.github/workflows/cluster_endtoend_23.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_24.yml b/.github/workflows/cluster_endtoend_24.yml index f56c3b89dc3..f0b6b4586bf 100644 --- a/.github/workflows/cluster_endtoend_24.yml +++ b/.github/workflows/cluster_endtoend_24.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_26.yml b/.github/workflows/cluster_endtoend_26.yml index ec5cdce36b9..72e40bf9da8 100644 --- a/.github/workflows/cluster_endtoend_26.yml +++ b/.github/workflows/cluster_endtoend_26.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_mysql80.yml b/.github/workflows/cluster_endtoend_mysql80.yml index a0b01bf7f4a..90bdd9da98a 100644 --- a/.github/workflows/cluster_endtoend_mysql80.yml +++ b/.github/workflows/cluster_endtoend_mysql80.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,16 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml index 2108309d473..2df2f9c77c0 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_declarative.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_declarative.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml index 87570937929..148c4afa32e 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_ghost.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_ghost.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_onlineddl_revert.yml b/.github/workflows/cluster_endtoend_onlineddl_revert.yml index 0f423454f60..78492eb15e6 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_revert.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_revert.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml index ba5681efc27..a9d1540a14a 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_singleton.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_singleton.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml index 0af5dd110c9..041799d55f7 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml index da4461bd75f..36c8c7fad9d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_stress.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml index 170ea0a38fd..ddcd6dd489d 100644 --- a/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml +++ b/.github/workflows/cluster_endtoend_onlineddl_vrepl_suite.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_resharding.yml b/.github/workflows/cluster_endtoend_resharding.yml index 53af8a07030..6a080f6ee81 100644 --- a/.github/workflows/cluster_endtoend_resharding.yml +++ b/.github/workflows/cluster_endtoend_resharding.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_resharding_bytes.yml b/.github/workflows/cluster_endtoend_resharding_bytes.yml index edec3162c4e..dbaa4a65ecb 100644 --- a/.github/workflows/cluster_endtoend_resharding_bytes.yml +++ b/.github/workflows/cluster_endtoend_resharding_bytes.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml index 5877c6462ed..5f2e0464ddd 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_tablegc.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml index e41b646eeba..bfc51a86dc4 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml index ac79b6a1d5b..09603534a59 100644 --- a/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml +++ b/.github/workflows/cluster_endtoend_tabletmanager_throttler_custom_config.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_upgrade.yml b/.github/workflows/cluster_endtoend_upgrade.yml index 306983854ec..7ff0b67a0f2 100644 --- a/.github/workflows/cluster_endtoend_upgrade.yml +++ b/.github/workflows/cluster_endtoend_upgrade.yml @@ -32,8 +32,6 @@ jobs: run: | # This prepares general purpose binary dependencies # as well as v9.0.0 specific go modules -<<<<<<< HEAD -======= sudo DEBIAN_FRONTEND="noninteractive" apt-get update @@ -47,10 +45,10 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* ->>>>>>> 630259cfeb32b85d5b53be3c68b555de710bbeb7 sudo apt-get update sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client diff --git a/.github/workflows/cluster_endtoend_vreplication_basic.yml b/.github/workflows/cluster_endtoend_vreplication_basic.yml index 8f4bd5b9e71..67eac94ebef 100644 --- a/.github/workflows/cluster_endtoend_vreplication_basic.yml +++ b/.github/workflows/cluster_endtoend_vreplication_basic.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml index b0ff10e203b..906a9fe3531 100644 --- a/.github/workflows/cluster_endtoend_vreplication_cellalias.yml +++ b/.github/workflows/cluster_endtoend_vreplication_cellalias.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vreplication_migrate.yml b/.github/workflows/cluster_endtoend_vreplication_migrate.yml index acef7c1bca6..267c78fcdef 100644 --- a/.github/workflows/cluster_endtoend_vreplication_migrate.yml +++ b/.github/workflows/cluster_endtoend_vreplication_migrate.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vreplication_multicell.yml b/.github/workflows/cluster_endtoend_vreplication_multicell.yml index 3250217f4fd..54810f35e76 100644 --- a/.github/workflows/cluster_endtoend_vreplication_multicell.yml +++ b/.github/workflows/cluster_endtoend_vreplication_multicell.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vreplication_v2.yml b/.github/workflows/cluster_endtoend_vreplication_v2.yml index c5337e4cc1d..289af923f9a 100644 --- a/.github/workflows/cluster_endtoend_vreplication_v2.yml +++ b/.github/workflows/cluster_endtoend_vreplication_v2.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_buffer.yml b/.github/workflows/cluster_endtoend_vtgate_buffer.yml index 93c4340c24a..206e2446669 100644 --- a/.github/workflows/cluster_endtoend_vtgate_buffer.yml +++ b/.github/workflows/cluster_endtoend_vtgate_buffer.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml index 299a01fad76..bd92c894917 100644 --- a/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml +++ b/.github/workflows/cluster_endtoend_vtgate_concurrentdml.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_gen4.yml b/.github/workflows/cluster_endtoend_vtgate_gen4.yml index 433a1e8c5b7..3110b421072 100644 --- a/.github/workflows/cluster_endtoend_vtgate_gen4.yml +++ b/.github/workflows/cluster_endtoend_vtgate_gen4.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml index 7ce88438886..308114c067e 100644 --- a/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml +++ b/.github/workflows/cluster_endtoend_vtgate_readafterwrite.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml index 593432bce46..e8c6337ced5 100644 --- a/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml +++ b/.github/workflows/cluster_endtoend_vtgate_reservedconn.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_schema.yml b/.github/workflows/cluster_endtoend_vtgate_schema.yml index 07a47fd64ff..5133776d0e1 100644 --- a/.github/workflows/cluster_endtoend_vtgate_schema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_schema.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_topo.yml b/.github/workflows/cluster_endtoend_vtgate_topo.yml index bd6a7d9f0dd..34b0179ee63 100644 --- a/.github/workflows/cluster_endtoend_vtgate_topo.yml +++ b/.github/workflows/cluster_endtoend_vtgate_topo.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_transaction.yml b/.github/workflows/cluster_endtoend_vtgate_transaction.yml index e8db2e3c009..d90fa9d8fd8 100644 --- a/.github/workflows/cluster_endtoend_vtgate_transaction.yml +++ b/.github/workflows/cluster_endtoend_vtgate_transaction.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml index a739f1f86bb..4067f84f56f 100644 --- a/.github/workflows/cluster_endtoend_vtgate_unsharded.yml +++ b/.github/workflows/cluster_endtoend_vtgate_unsharded.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_vindex.yml b/.github/workflows/cluster_endtoend_vtgate_vindex.yml index e6d43c31d51..6df8b8d051b 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vindex.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vindex.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtgate_vschema.yml b/.github/workflows/cluster_endtoend_vtgate_vschema.yml index 60360c49781..0d8b12e0108 100644 --- a/.github/workflows/cluster_endtoend_vtgate_vschema.yml +++ b/.github/workflows/cluster_endtoend_vtgate_vschema.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_vtorc.yml b/.github/workflows/cluster_endtoend_vtorc.yml index 98e77b4fe92..3f53de97376 100644 --- a/.github/workflows/cluster_endtoend_vtorc.yml +++ b/.github/workflows/cluster_endtoend_vtorc.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/cluster_endtoend_xb_recovery.yml b/.github/workflows/cluster_endtoend_xb_recovery.yml index f56dc126f44..9f009fe4493 100644 --- a/.github/workflows/cluster_endtoend_xb_recovery.yml +++ b/.github/workflows/cluster_endtoend_xb_recovery.yml @@ -17,6 +17,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -29,8 +32,11 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index 6c9901bfdc3..61e706f3ca3 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -34,6 +34,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/legacy_local_example.yml b/.github/workflows/legacy_local_example.yml index b7c2f8145f2..62e83d83877 100644 --- a/.github/workflows/legacy_local_example.yml +++ b/.github/workflows/legacy_local_example.yml @@ -40,6 +40,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index e9e1430704a..c45f8a917cb 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -40,6 +40,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/region_example.yml b/.github/workflows/region_example.yml index a50784718d0..d288c18bfef 100644 --- a/.github/workflows/region_example.yml +++ b/.github/workflows/region_example.yml @@ -40,6 +40,7 @@ jobs: sudo rm -rf /etc/mysql # Install mysql80 + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* diff --git a/.github/workflows/unit_test_mysql80.yml b/.github/workflows/unit_test_mysql80.yml index 6fa7eca1161..f79a86f563e 100644 --- a/.github/workflows/unit_test_mysql80.yml +++ b/.github/workflows/unit_test_mysql80.yml @@ -42,6 +42,9 @@ jobs: sudo rm -rf /var/lib/mysql sudo rm -rf /etc/mysql + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # mysql80 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections diff --git a/Makefile b/Makefile index 94ecd5f1772..91120cfa11b 100644 --- a/Makefile +++ b/Makefile @@ -145,8 +145,8 @@ sizegen: go run ./go/tools/sizegen/sizegen.go \ -in ./go/... \ -gen vitess.io/vitess/go/vt/vtgate/engine.Plan \ - -gen vitess.io/vitess/go/vt/vttablet/tabletserver.TabletPlan \ - -gen vitess.io/vitess/go/sqltypes.Result + -gen vitess.io/vitess/go/vt/vttablet/tabletserver.TabletPlan \ + -gen vitess.io/vitess/go/sqltypes.Result astfmtgen: go run ./go/tools/astfmtgen/main.go vitess.io/vitess/go/vt/sqlparser/... diff --git a/docker/bootstrap/Dockerfile.mysql56 b/docker/bootstrap/Dockerfile.mysql56 index 7cfb8e6876f..1ce1229de37 100644 --- a/docker/bootstrap/Dockerfile.mysql56 +++ b/docker/bootstrap/Dockerfile.mysql56 @@ -11,6 +11,7 @@ FROM "${image}" # I think it's fine as MySQL 5.6 will be EOL pretty soon (February 5, 2021) # RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \ + for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 467B942D3A79BD29 && break; done && \ add-apt-repository 'deb http://repo.mysql.com/apt/debian/ stretch mysql-5.6' && \ for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \ echo 'deb http://repo.percona.com/apt buster main' > /etc/apt/sources.list.d/percona.list && \ diff --git a/docker/bootstrap/Dockerfile.mysql57 b/docker/bootstrap/Dockerfile.mysql57 index 58565b1a377..0b8cf107aed 100644 --- a/docker/bootstrap/Dockerfile.mysql57 +++ b/docker/bootstrap/Dockerfile.mysql57 @@ -6,6 +6,7 @@ FROM "${image}" # Install MySQL 5.7 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \ for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \ + for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 467B942D3A79BD29 && break; done && \ add-apt-repository 'deb http://repo.mysql.com/apt/debian/ buster mysql-5.7' && \ for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \ echo 'deb http://repo.percona.com/apt buster main' > /etc/apt/sources.list.d/percona.list && \ diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index ab1c9b97898..3ae0a3b63b7 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -5,6 +5,7 @@ FROM "${image}" # Install MySQL 8.0 RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 8C718D3B5072E1F5 && break; done && \ + for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 467B942D3A79BD29 && break; done && \ add-apt-repository 'deb http://repo.mysql.com/apt/debian/ buster mysql-8.0' && \ for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keyserver.ubuntu.com --recv-keys 9334A25F8507EFA5 && break; done && \ echo 'deb http://repo.percona.com/apt buster main' > /etc/apt/sources.list.d/percona.list && \ diff --git a/docker/lite/install_dependencies.sh b/docker/lite/install_dependencies.sh index ac257d871fb..b88763f31d0 100755 --- a/docker/lite/install_dependencies.sh +++ b/docker/lite/install_dependencies.sh @@ -156,6 +156,7 @@ case "${FLAVOR}" in mysql56|mysql57|mysql80) # repo.mysql.com add_apt_key 8C718D3B5072E1F5 + add_apt_key 467B942D3A79BD29 ;; mariadb|mariadb103) # digitalocean.com diff --git a/go.mod b/go.mod index 1a8ee45a9f7..6037ff64afa 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/aquarapid/vaultlib v0.5.1 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect - github.com/aws/aws-sdk-go v1.28.8 + github.com/aws/aws-sdk-go v1.34.2 github.com/buger/jsonparser v0.0.0-20200322175846-f7e751efca13 github.com/cespare/xxhash/v2 v2.1.1 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect @@ -47,6 +47,7 @@ require ( github.com/howeyc/gopass v0.0.0-20190910152052-7cb4b85ec19c github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428 github.com/imdario/mergo v0.3.6 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmoiron/sqlx v1.2.0 github.com/klauspost/compress v1.4.1 // indirect github.com/klauspost/cpuid v1.2.0 // indirect diff --git a/go.sum b/go.sum index 5956a7cd65e..93511fbfb0a 100644 --- a/go.sum +++ b/go.sum @@ -115,8 +115,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.28.8 h1:kPGnElMdW0GDc54Giy1lcE/3gAr2Gzl6cMjYKoBNFhw= -github.com/aws/aws-sdk-go v1.28.8/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.34.2 h1:9vCknCdTAmmV4ht7lPuda7aJXzllXwEQyCMZKJHjBrM= +github.com/aws/aws-sdk-go v1.34.2/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -259,6 +259,7 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+ github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.5.1-0.20210202043019-fe2230a8b20c h1:yUT3Ygm3yXBD2qLPxYRDBcnEz0MHgQ4TJ/87C/wKnWA= github.com/go-sql-driver/mysql v1.5.1-0.20210202043019-fe2230a8b20c/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -441,8 +442,11 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= diff --git a/go/test/endtoend/vtgate/system_schema_test.go b/go/test/endtoend/vtgate/system_schema_test.go index c5b0af8e445..ad2fb0f5e93 100644 --- a/go/test/endtoend/vtgate/system_schema_test.go +++ b/go/test/endtoend/vtgate/system_schema_test.go @@ -214,5 +214,5 @@ func TestMultipleSchemaPredicates(t *testing.T) { "where t.table_schema = '%s' and c.table_schema = '%s' and c.table_schema = '%s'", KeyspaceName, KeyspaceName, "a") _, err = conn.ExecuteFetch(query, 1000, true) require.Error(t, err) - require.Contains(t, err.Error(), "two predicates for specifying the database are not supported") + require.Contains(t, err.Error(), "specifying two different database in the query is not supported") } diff --git a/go/vt/sqlparser/sql.y b/go/vt/sqlparser/sql.y index 9f42384667d..bc6c3029335 100644 --- a/go/vt/sqlparser/sql.y +++ b/go/vt/sqlparser/sql.y @@ -5326,4 +5326,4 @@ ddl_skip_to_end: | reserved_sql_id { skipToEnd(yylex) - } \ No newline at end of file + } diff --git a/go/vt/vtctld/vtctld.go b/go/vt/vtctld/vtctld.go index 66efd262f68..646bf49db50 100644 --- a/go/vt/vtctld/vtctld.go +++ b/go/vt/vtctld/vtctld.go @@ -19,6 +19,7 @@ limitations under the License. package vtctld import ( + "context" "flag" "net/http" "strings" @@ -26,8 +27,6 @@ import ( rice "github.com/GeertJohan/go.rice" - "context" - "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/acl" @@ -39,6 +38,7 @@ import ( var ( enableRealtimeStats = flag.Bool("enable_realtime_stats", false, "Required for the Realtime Stats view. If set, vtctld will maintain a streaming RPC to each tablet (in all cells) to gather the realtime health stats.") + sanitizeLogMessages = flag.Bool("vtctld_sanitize_log_messages", false, "When true, vtctld sanitizes logging.") _ = flag.String("web_dir", "", "NOT USED, here for backward compatibility") _ = flag.String("web_dir2", "", "NOT USED, here for backward compatibility") diff --git a/go/vt/vtctld/workflow.go b/go/vt/vtctld/workflow.go index da815a69c07..5656c2dc6ee 100644 --- a/go/vt/vtctld/workflow.go +++ b/go/vt/vtctld/workflow.go @@ -17,11 +17,10 @@ limitations under the License. package vtctld import ( + "context" "flag" "time" - "context" - "vitess.io/vitess/go/trace" "vitess.io/vitess/go/flagutil" @@ -68,6 +67,7 @@ func initWorkflowManager(ts *topo.Server) { // Create the WorkflowManager. vtctl.WorkflowManager = workflow.NewManager(ts) + vtctl.WorkflowManager.SetSanitizeHTTPHeaders(*sanitizeLogMessages) // Register the long polling and websocket handlers. vtctl.WorkflowManager.HandleHTTPLongPolling(apiPrefix + "workflow") diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index fd600cb2588..bd78144b732 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -217,29 +217,10 @@ func TestOne(t *testing.T) { testFile(t, "onecase.txt", "", vschema, true) } -func TestBypassPlanningShardTargetFromFile(t *testing.T) { +func TestBypassPlanningFromFile(t *testing.T) { testOutputTempDir, err := ioutil.TempDir("", "plan_test") require.NoError(t, err) defer os.RemoveAll(testOutputTempDir) - - vschema := &vschemaWrapper{ - v: loadSchema(t, "schema_test.json"), - keyspace: &vindexes.Keyspace{ - Name: "main", - Sharded: false, - }, - tabletType: topodatapb.TabletType_MASTER, - dest: key.DestinationShard("-80")} - - testFile(t, "bypass_shard_cases.txt", testOutputTempDir, vschema, true) -} -func TestBypassPlanningKeyrangeTargetFromFile(t *testing.T) { - testOutputTempDir, err := ioutil.TempDir("", "plan_test") - require.NoError(t, err) - defer os.RemoveAll(testOutputTempDir) - - keyRange, _ := key.ParseShardingSpec("-") - vschema := &vschemaWrapper{ v: loadSchema(t, "schema_test.json"), keyspace: &vindexes.Keyspace{ @@ -247,10 +228,10 @@ func TestBypassPlanningKeyrangeTargetFromFile(t *testing.T) { Sharded: false, }, tabletType: topodatapb.TabletType_MASTER, - dest: key.DestinationExactKeyRange{KeyRange: keyRange[0]}, + dest: key.DestinationShard("-80"), } - testFile(t, "bypass_keyrange_cases.txt", testOutputTempDir, vschema, true) + testFile(t, "bypass_cases.txt", testOutputTempDir, vschema, true) } func TestWithDefaultKeyspaceFromFile(t *testing.T) { diff --git a/go/vt/vtgate/planbuilder/testdata/bypass_keyrange_cases.txt b/go/vt/vtgate/planbuilder/testdata/bypass_keyrange_cases.txt deleted file mode 100644 index 3b121cc2cd0..00000000000 --- a/go/vt/vtgate/planbuilder/testdata/bypass_keyrange_cases.txt +++ /dev/null @@ -1,174 +0,0 @@ -# select bypass -"select count(*), col from unsharded" -{ - "QueryType": "SELECT", - "Original": "select count(*), col from unsharded", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": false, - "Query": "select count(*), col from unsharded", - "SingleShardOnly": false - } -} -Gen4 plan same as above - -# update bypass -"update user set val = 1 where id = 18446744073709551616 and id = 1" -{ - "QueryType": "UPDATE", - "Original": "update user set val = 1 where id = 18446744073709551616 and id = 1", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": true, - "Query": "update `user` set val = 1 where id = 18446744073709551616 and id = 1", - "SingleShardOnly": false - } -} -Gen4 plan same as above - -# update bypass autocommit -"update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ user set val = 1 where id = 18446744073709551616 and id = 1" -{ - "QueryType": "UPDATE", - "Original": "update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ user set val = 1 where id = 18446744073709551616 and id = 1", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": true, - "MultishardAutocommit": true, - "Query": "update /*vt+ MULTI_SHARD_AUTOCOMMIT=1 */ `user` set val = 1 where id = 18446744073709551616 and id = 1", - "SingleShardOnly": false - } -} -Gen4 plan same as above - -# delete bypass -"DELETE FROM USER WHERE ID = 42" -{ - "QueryType": "DELETE", - "Original": "DELETE FROM USER WHERE ID = 42", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": true, - "Query": "delete from `USER` where ID = 42", - "SingleShardOnly": false - } -} -Gen4 plan same as above - -# insert bypass: not supported -"INSERT INTO USER (ID, NAME) VALUES (42, 'ms X')" -"range queries are not allowed for insert statement: targetString" -Gen4 plan same as above - -# bypass query for into outfile s3 -"select count(*), col from unsharded into outfile S3 'x.txt'" -{ - "QueryType": "SELECT", - "Original": "select count(*), col from unsharded into outfile S3 'x.txt'", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": false, - "Query": "select count(*), col from unsharded into outfile s3 'x.txt'", - "SingleShardOnly": false - } -} -Gen4 plan same as above - -# Select outfile -"select * from user into outfile S3 'x.txt'" -{ - "QueryType": "SELECT", - "Original": "select * from user into outfile S3 'x.txt'", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": false, - "Query": "select * from `user` into outfile s3 'x.txt'", - "SingleShardOnly": false - } -} -Gen4 plan same as above - -"load data from s3 'x.txt' into table x" -{ - "QueryType": "OTHER", - "Original": "load data from s3 'x.txt' into table x", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": true, - "Query": "load data from s3 'x.txt' into table x", - "SingleShardOnly": true - } -} -Gen4 plan same as above - -"load data from s3 'x.txt'" -{ - "QueryType": "OTHER", - "Original": "load data from s3 'x.txt'", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": true, - "Query": "load data from s3 'x.txt'", - "SingleShardOnly": true - } -} -Gen4 plan same as above - -# create table -"create /* test */ table t1(id bigint, primary key(id)) /* comments */" -{ - "QueryType": "DDL", - "Original": "create /* test */ table t1(id bigint, primary key(id)) /* comments */", - "Instructions": { - "OperatorType": "Send", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "TargetDestination": "ExactKeyRange(-)", - "IsDML": false, - "Query": "create /* test */ table t1(id bigint, primary key(id)) /* comments */", - "SingleShardOnly": false - } -} -Gen4 plan same as above diff --git a/go/vt/vtgate/planbuilder/testdata/filter_cases.txt b/go/vt/vtgate/planbuilder/testdata/filter_cases.txt index efaeefa33f2..c7642bf5369 100644 --- a/go/vt/vtgate/planbuilder/testdata/filter_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/filter_cases.txt @@ -1913,7 +1913,21 @@ Gen4 plan same as above # query trying to query two different keyspaces at the same time "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'user' AND TABLE_SCHEMA = 'main'" -"two predicates for specifying the database are not supported" +{ + "QueryType": "SELECT", + "Original": "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'user' AND TABLE_SCHEMA = 'main'", + "Instructions": { + "OperatorType": "Route", + "Variant": "SelectDBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select * from INFORMATION_SCHEMA.`TABLES` where 1 != 1", + "Query": "select * from INFORMATION_SCHEMA.`TABLES` where TABLE_SCHEMA = :__vtschemaname and TABLE_SCHEMA = :__vtschemaname", + "SysTableTableSchema": "[VARBINARY(\"user\"), VARBINARY(\"main\")]" + } +} # information_schema query using database() func "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = database()" @@ -1946,7 +1960,7 @@ Gen4 plan same as above }, "FieldQuery": "select * from INFORMATION_SCHEMA.`TABLES` where 1 != 1", "Query": "select * from INFORMATION_SCHEMA.`TABLES` where TABLE_SCHEMA = :__vtschemaname", - "SysTableTableSchema": "VARBINARY(\"ks\")" + "SysTableTableSchema": "[VARBINARY(\"ks\")]" } } @@ -1981,8 +1995,8 @@ Gen4 plan same as above }, "FieldQuery": "select * from INFORMATION_SCHEMA.`TABLES` where 1 != 1", "Query": "select * from INFORMATION_SCHEMA.`TABLES` where TABLE_SCHEMA = :__vtschemaname and TABLE_NAME = :__vttablename", - "SysTableTableName": "VARBINARY(\"route1\")", - "SysTableTableSchema": "VARBINARY(\"ks\")" + "SysTableTableName": "[VARBINARY(\"route1\")]", + "SysTableTableSchema": "[VARBINARY(\"ks\")]" } } diff --git a/go/vt/vtgate/planbuilder/testdata/from_cases.txt b/go/vt/vtgate/planbuilder/testdata/from_cases.txt index 7c86f92c4d2..3a0e1b308de 100644 --- a/go/vt/vtgate/planbuilder/testdata/from_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/from_cases.txt @@ -2767,34 +2767,15 @@ Gen4 plan same as above "QueryType": "SELECT", "Original": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as name, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc join information_schema.key_column_usage as fk using (constraint_schema, constraint_name) where fk.referenced_column_name is not null and fk.table_schema = database() and fk.table_name = ':vtg1' and rc.constraint_schema = database() and rc.table_name = ':vtg1'", "Instructions": { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "1,2,3,4,-1,-2", - "TableName": "_", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectDBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select rc.update_rule as on_update, rc.delete_rule as on_delete, rc.constraint_schema, rc.constraint_name from information_schema.referential_constraints as rc where 1 != 1", - "Query": "select rc.update_rule as on_update, rc.delete_rule as on_delete, rc.constraint_schema, rc.constraint_name from information_schema.referential_constraints as rc where rc.constraint_schema = database() and rc.table_name = :__vttablename", - "SysTableTableName": "VARBINARY(\":vtg1\")" - }, - { - "OperatorType": "Route", - "Variant": "SelectDBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name` from information_schema.key_column_usage as fk where 1 != 1", - "Query": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name` from information_schema.key_column_usage as fk where fk.constraint_schema = :rc_constraint_schema and fk.constraint_name = :rc_constraint_name and fk.referenced_column_name is not null and fk.table_schema = database() and fk.table_name = :__vttablename", - "SysTableTableName": "VARBINARY(\":vtg1\")" - } - ] + "OperatorType": "Route", + "Variant": "SelectDBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc join information_schema.key_column_usage as fk on rc.constraint_schema = fk.constraint_schema and rc.constraint_name = fk.constraint_name where 1 != 1", + "Query": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc join information_schema.key_column_usage as fk on rc.constraint_schema = fk.constraint_schema and rc.constraint_name = fk.constraint_name where fk.referenced_column_name is not null and fk.table_schema = database() and fk.table_name = :__vttablename and rc.constraint_schema = database() and rc.table_name = :__vttablename", + "SysTableTableName": "[VARBINARY(\":vtg1\"), VARBINARY(\":vtg1\")]" } } @@ -2812,7 +2793,7 @@ Gen4 plan same as above }, "FieldQuery": "select * from information_schema.schemata where 1 != 1", "Query": "select * from information_schema.schemata where schema_name = :__vtschemaname", - "SysTableTableSchema": "VARBINARY(\"user\")" + "SysTableTableSchema": "[VARBINARY(\"user\")]" } } @@ -2830,8 +2811,8 @@ Gen4 plan same as above }, "FieldQuery": "select table_comment from information_schema.`tables` where 1 != 1", "Query": "select table_comment from information_schema.`tables` where table_schema = :__vtschemaname and table_name = :__vttablename", - "SysTableTableName": "VARBINARY(\"table_name\")", - "SysTableTableSchema": "VARBINARY(\"schema_name\")" + "SysTableTableName": "[VARBINARY(\"table_name\")]", + "SysTableTableSchema": "[VARBINARY(\"schema_name\")]" } } @@ -2841,36 +2822,16 @@ Gen4 plan same as above "QueryType": "SELECT", "Original": "SELECT fk.referenced_table_name AS 'to_table', fk.referenced_column_name AS 'primary_key',fk.column_name AS 'column',fk.constraint_name AS 'name',rc.update_rule AS 'on_update',rc.delete_rule AS 'on_delete' FROM information_schema.referential_constraints rc JOIN information_schema.key_column_usage fk USING (constraint_schema, constraint_name) WHERE fk.referenced_column_name IS NOT NULL AND fk.table_schema = 'table_schema' AND fk.table_name = 'table_name' AND rc.constraint_schema = 'table_schema' AND rc.table_name = 'table_name'", "Instructions": { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "1,2,3,4,-1,-2", - "TableName": "_", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectDBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select rc.update_rule as on_update, rc.delete_rule as on_delete, rc.constraint_schema, rc.constraint_name from information_schema.referential_constraints as rc where 1 != 1", - "Query": "select rc.update_rule as on_update, rc.delete_rule as on_delete, rc.constraint_schema, rc.constraint_name from information_schema.referential_constraints as rc where rc.constraint_schema = :__vtschemaname and rc.table_name = :__vttablename", - "SysTableTableName": "VARBINARY(\"table_name\")", - "SysTableTableSchema": "VARBINARY(\"table_schema\")" - }, - { - "OperatorType": "Route", - "Variant": "SelectDBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name` from information_schema.key_column_usage as fk where 1 != 1", - "Query": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name` from information_schema.key_column_usage as fk where fk.constraint_schema = :rc_constraint_schema and fk.constraint_name = :rc_constraint_name and fk.referenced_column_name is not null and fk.table_schema = :__vtschemaname and fk.table_name = :__vttablename", - "SysTableTableName": "VARBINARY(\"table_name\")", - "SysTableTableSchema": "VARBINARY(\"table_schema\")" - } - ] + "OperatorType": "Route", + "Variant": "SelectDBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc join information_schema.key_column_usage as fk on rc.constraint_schema = fk.constraint_schema and rc.constraint_name = fk.constraint_name where 1 != 1", + "Query": "select fk.referenced_table_name as to_table, fk.referenced_column_name as primary_key, fk.column_name as `column`, fk.constraint_name as `name`, rc.update_rule as on_update, rc.delete_rule as on_delete from information_schema.referential_constraints as rc join information_schema.key_column_usage as fk on rc.constraint_schema = fk.constraint_schema and rc.constraint_name = fk.constraint_name where fk.referenced_column_name is not null and fk.table_schema = :__vtschemaname and fk.table_name = :__vttablename and rc.constraint_schema = :__vtschemaname and rc.table_name = :__vttablename", + "SysTableTableName": "[VARBINARY(\"table_name\"), VARBINARY(\"table_name\")]", + "SysTableTableSchema": "[VARBINARY(\"table_schema\"), VARBINARY(\"table_schema\")]" } } @@ -2880,35 +2841,16 @@ Gen4 plan same as above "QueryType": "SELECT", "Original": "SELECT cc.constraint_name AS 'name', cc.check_clause AS 'expression' FROM information_schema.check_constraints cc JOIN information_schema.table_constraints tc USING (constraint_schema, constraint_name) WHERE tc.table_schema = 'table_schema' AND tc.table_name = 'table_name' AND cc.constraint_schema = 'constraint_schema'", "Instructions": { - "OperatorType": "Join", - "Variant": "Join", - "JoinColumnIndexes": "-1,-2", - "TableName": "_", - "Inputs": [ - { - "OperatorType": "Route", - "Variant": "SelectDBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select cc.constraint_name as `name`, cc.check_clause as expression, cc.constraint_schema from information_schema.check_constraints as cc where 1 != 1", - "Query": "select cc.constraint_name as `name`, cc.check_clause as expression, cc.constraint_schema from information_schema.check_constraints as cc where cc.constraint_schema = :__vtschemaname", - "SysTableTableSchema": "VARBINARY(\"constraint_schema\")" - }, - { - "OperatorType": "Route", - "Variant": "SelectDBA", - "Keyspace": { - "Name": "main", - "Sharded": false - }, - "FieldQuery": "select 1 from information_schema.table_constraints as tc where 1 != 1", - "Query": "select 1 from information_schema.table_constraints as tc where tc.constraint_schema = :cc_constraint_schema and tc.constraint_name = :cc_constraint_name and tc.table_schema = :__vtschemaname and tc.table_name = :__vttablename", - "SysTableTableName": "VARBINARY(\"table_name\")", - "SysTableTableSchema": "VARBINARY(\"table_schema\")" - } - ] + "OperatorType": "Route", + "Variant": "SelectDBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select cc.constraint_name as `name`, cc.check_clause as expression from information_schema.check_constraints as cc join information_schema.table_constraints as tc on cc.constraint_schema = tc.constraint_schema and cc.constraint_name = tc.constraint_name where 1 != 1", + "Query": "select cc.constraint_name as `name`, cc.check_clause as expression from information_schema.check_constraints as cc join information_schema.table_constraints as tc on cc.constraint_schema = tc.constraint_schema and cc.constraint_name = tc.constraint_name where tc.table_schema = :__vtschemaname and tc.table_name = :__vttablename and cc.constraint_schema = :__vtschemaname", + "SysTableTableName": "[VARBINARY(\"table_name\")]", + "SysTableTableSchema": "[VARBINARY(\"table_schema\"), VARBINARY(\"constraint_schema\")]" } } @@ -2926,8 +2868,8 @@ Gen4 plan same as above }, "FieldQuery": "select column_name from information_schema.statistics where 1 != 1", "Query": "select column_name from information_schema.statistics where index_name = 'PRIMARY' and table_schema = :__vtschemaname and table_name = :__vttablename order by seq_in_index asc", - "SysTableTableName": "VARBINARY(\"table_name\")", - "SysTableTableSchema": "VARBINARY(\"table_schema\")" + "SysTableTableName": "[VARBINARY(\"table_name\")]", + "SysTableTableSchema": "[VARBINARY(\"table_schema\")]" } } @@ -2945,8 +2887,8 @@ Gen4 plan same as above }, "FieldQuery": "select generation_expression from information_schema.`columns` where 1 != 1", "Query": "select generation_expression from information_schema.`columns` where table_schema = :__vtschemaname and table_name = :__vttablename and column_name = 'column_name'", - "SysTableTableName": "VARBINARY(\"table_name\")", - "SysTableTableSchema": "VARBINARY(\"table_schema\")" + "SysTableTableName": "[VARBINARY(\"table_name\")]", + "SysTableTableSchema": "[VARBINARY(\"table_schema\")]" } } @@ -2981,7 +2923,7 @@ Gen4 plan same as above }, "FieldQuery": "select table_name from (select * from information_schema.`tables` where 1 != 1) as _subquery where 1 != 1", "Query": "select table_name from (select * from information_schema.`tables` where table_schema = :__vtschemaname) as _subquery", - "SysTableTableSchema": "VARBINARY(\"table_schema\")" + "SysTableTableSchema": "[VARBINARY(\"table_schema\")]" } } @@ -2999,8 +2941,8 @@ Gen4 plan same as above }, "FieldQuery": "select table_name from (select * from information_schema.`tables` where 1 != 1) as _subquery where 1 != 1", "Query": "select table_name from (select * from information_schema.`tables` where table_schema = :__vtschemaname) as _subquery where _subquery.table_type = 'table_type' and _subquery.table_name = :__vttablename", - "SysTableTableName": "VARBINARY(\"table_name\")", - "SysTableTableSchema": "VARBINARY(\"table_schema\")" + "SysTableTableName": "[VARBINARY(\"table_name\")]", + "SysTableTableSchema": "[VARBINARY(\"table_schema\")]" } } @@ -3018,7 +2960,7 @@ Gen4 plan same as above }, "FieldQuery": "select cc.constraint_name as `name` from information_schema.check_constraints as cc where 1 != 1", "Query": "select cc.constraint_name as `name` from information_schema.check_constraints as cc where cc.constraint_schema = :__vtschemaname and cc.table_schema = :__vtschemaname", - "SysTableTableSchema": "VARBINARY(\"a\")" + "SysTableTableSchema": "[VARBINARY(\"a\"), VARBINARY(\"a\")]" } } diff --git a/go/vt/vtgate/planbuilder/testdata/select_cases.txt b/go/vt/vtgate/planbuilder/testdata/select_cases.txt index 51ebd118883..46650433aeb 100644 --- a/go/vt/vtgate/planbuilder/testdata/select_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/select_cases.txt @@ -1802,4 +1802,4 @@ Gen4 plan same as above "Query": "select kcu.constraint_name as constraint_name, kcu.column_name as column_name, kcu.referenced_table_name as referenced_table_name, kcu.referenced_column_name as referenced_column_name, kcu.ordinal_position as ordinal_position, kcu.table_name as table_name, rc.delete_rule as delete_rule, rc.update_rule as update_rule from information_schema.key_column_usage as kcu join information_schema.referential_constraints as rc on kcu.constraint_name = rc.constraint_name where kcu.table_schema = :__vtschemaname and rc.constraint_schema = :__vtschemaname and kcu.referenced_column_name is not null order by ordinal_position asc", "SysTableTableSchema": "[:v1, :v2]" } -} \ No newline at end of file +} diff --git a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt index 48a2751778c..19e52bc5890 100644 --- a/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt +++ b/go/vt/vtgate/planbuilder/testdata/unsupported_cases.txt @@ -425,8 +425,23 @@ Gen4 plan same as above "INTO is not supported on sharded keyspace" # unsupported two predicates specifying the database for the same table if they are different +# will fail on run time but will pass on the planbuilder "SELECT cc.constraint_name AS 'name' FROM information_schema.check_constraints cc WHERE cc.constraint_schema = 'constraint_schema' AND cc.table_schema = 'a'" -"two predicates for specifying the database are not supported" +{ + "QueryType": "SELECT", + "Original": "SELECT cc.constraint_name AS 'name' FROM information_schema.check_constraints cc WHERE cc.constraint_schema = 'constraint_schema' AND cc.table_schema = 'a'", + "Instructions": { + "OperatorType": "Route", + "Variant": "SelectDBA", + "Keyspace": { + "Name": "main", + "Sharded": false + }, + "FieldQuery": "select cc.constraint_name as `name` from information_schema.check_constraints as cc where 1 != 1", + "Query": "select cc.constraint_name as `name` from information_schema.check_constraints as cc where cc.constraint_schema = :__vtschemaname and cc.table_schema = :__vtschemaname", + "SysTableTableSchema": "[VARBINARY(\"constraint_schema\"), VARBINARY(\"a\")]" + } +} # create view with Cannot auto-resolve for cross-shard joins "create view user.view_a as select col from user join user_extra" diff --git a/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt b/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt index ed3e3357fd5..e7ec2cd817d 100644 --- a/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt +++ b/go/vt/vttablet/tabletserver/planbuilder/testdata/exec_cases.txt @@ -914,20 +914,6 @@ options:PassthroughDMLs "FullQuery":"drop view a, b" } -# create table with function as a default value -"create table function_default (x varchar(25) DEFAULT (TRIM(' check ')))" -{ - "PlanID": "DDL", - "TableName": "", - "Permissions": [ - { - "TableName": "function_default", - "Role": 2 - } - ], - "FullQuery": "create table function_default (\n\tx varchar(25) default TRIM(' check ')\n)" -} - # flush statement "flush tables a,b" { diff --git a/go/vt/workflow/long_polling.go b/go/vt/workflow/long_polling.go index 3e804a8ff63..963ed624ec0 100644 --- a/go/vt/workflow/long_polling.go +++ b/go/vt/workflow/long_polling.go @@ -17,6 +17,7 @@ limitations under the License. package workflow import ( + "context" "encoding/json" "fmt" "io/ioutil" @@ -27,8 +28,6 @@ import ( "sync" "time" - "context" - "vitess.io/vitess/go/acl" "vitess.io/vitess/go/timer" "vitess.io/vitess/go/vt/log" @@ -157,27 +156,62 @@ func getID(url, base string) (int, error) { return i, nil } -func httpErrorf(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) { +const HTTPHeaderRedactedMessage string = "*** redacted by sanitizeRequestHeader() ***" + +var allowedHTTPHeadersList = map[string]interface{}{ + "Accept": nil, + "Accept-Encoding": nil, + "Accept-Language": nil, + "Upgrade-Insecure-Requests": nil, + "User-Agent": nil, + "X-Forwarded-For": nil, + "X-Forwarded-Host": nil, + "X-Forwarded-Proto": nil, + "X-Real-Ip": nil, +} + +// sanitizeRequestHeader - (unless sanitizeHTTPHeaders=false) makes a copy of r and returns it with sanitized headers +func sanitizeRequestHeader(r *http.Request, sanitizeHTTPHeaders bool) *http.Request { + if !sanitizeHTTPHeaders { + return r + } + + s := r.Clone(r.Context()) + + for k := range s.Header { + if _, ok := allowedHTTPHeadersList[k]; !ok { + s.Header.Set(k, HTTPHeaderRedactedMessage) + } + } + + return s +} + +func (m *Manager) httpErrorf(w http.ResponseWriter, r *http.Request, format string, args ...interface{}) { errMsg := fmt.Sprintf(format, args...) - log.Errorf("HTTP error on %v: %v, request: %#v", r.URL.Path, errMsg, r) + log.Errorf("HTTP error on %v: %v, request: %#v", + r.URL.Path, + errMsg, + sanitizeRequestHeader(r, m.sanitizeHTTPHeaders), + ) http.Error(w, errMsg, http.StatusInternalServerError) } -func handleAPI(pattern string, handlerFunc func(w http.ResponseWriter, r *http.Request) error) { +func (m *Manager) handleAPI(pattern string, handlerFunc func(w http.ResponseWriter, r *http.Request) error) { http.HandleFunc(pattern, func(w http.ResponseWriter, r *http.Request) { defer func() { if x := recover(); x != nil { - httpErrorf(w, r, "uncaught panic: %v", x) + m.httpErrorf(w, r, "uncaught panic: %v", x) } }() if err := acl.CheckAccessHTTP(r, acl.ADMIN); err != nil { - httpErrorf(w, r, "WorkflowManager acl.CheckAccessHTTP failed: %v", err) + m.httpErrorf(w, r, "WorkflowManager acl.CheckAccessHTTP failed: %v", err) return } if err := handlerFunc(w, r); err != nil { - httpErrorf(w, r, "%v", err) + m.httpErrorf(w, r, "%v", err) } }) } @@ -187,7 +221,7 @@ func (m *Manager) HandleHTTPLongPolling(pattern string) { log.Infof("workflow Manager listening to web traffic at %v/{create,poll,delete}", pattern) lpm := newLongPollingManager(m) - handleAPI(pattern+"/create", func(w http.ResponseWriter, r *http.Request) error { + m.handleAPI(pattern+"/create", func(w http.ResponseWriter, r *http.Request) error { result, i, err := lpm.create(r.URL) if err != nil { return fmt.Errorf("longPollingManager.create failed: %v", err) @@ -201,7 +235,7 @@ func (m *Manager) HandleHTTPLongPolling(pattern string) { return nil }) - handleAPI(pattern+"/poll/", func(w http.ResponseWriter, r *http.Request) error { + m.handleAPI(pattern+"/poll/", func(w http.ResponseWriter, r *http.Request) error { i, err := getID(r.URL.Path, pattern+"/poll/") if err != nil { return err @@ -220,7 +254,7 @@ func (m *Manager) HandleHTTPLongPolling(pattern string) { return nil }) - handleAPI(pattern+"/remove/", func(w http.ResponseWriter, r *http.Request) error { + m.handleAPI(pattern+"/remove/", func(w http.ResponseWriter, r *http.Request) error { i, err := getID(r.URL.Path, pattern+"/remove/") if err != nil { return err @@ -231,7 +265,7 @@ func (m *Manager) HandleHTTPLongPolling(pattern string) { return nil }) - handleAPI(pattern+"/action/", func(w http.ResponseWriter, r *http.Request) error { + m.handleAPI(pattern+"/action/", func(w http.ResponseWriter, r *http.Request) error { _, err := getID(r.URL.Path, pattern+"/action/") if err != nil { return err diff --git a/go/vt/workflow/long_polling_test.go b/go/vt/workflow/long_polling_test.go index 691a7a145bd..595bf3b993a 100644 --- a/go/vt/workflow/long_polling_test.go +++ b/go/vt/workflow/long_polling_test.go @@ -18,10 +18,12 @@ package workflow import ( "bytes" + "context" "io/ioutil" "net" "net/http" "net/url" + "reflect" "strings" "testing" "time" @@ -132,3 +134,87 @@ func TestLongPolling(t *testing.T) { cancel() wg.Wait() } + +func TestSanitizeRequestHeader(t *testing.T) { + testCases := []struct { + name string + sanitizeHTTPHeaders bool + reqMethod string + reqURL string + reqHeader map[string]string + wantHeader http.Header + }{ + { + name: "withCookie-and-sanitize", + sanitizeHTTPHeaders: true, + reqMethod: "GET", + reqURL: "https://vtctld-dev-xyz.company.com/api/workflow/poll/11", + reqHeader: map[string]string{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp", + "Cookie": "_ga=GA1.2.1234567899.1234567890; machine-cookie=username:123456789:1234f999ff99aa1af3ae9999d2a5d3968ac014a0947ae1418b10642ab2cbb7d3; session=7e5dc76807be39dd0886b9425883dabe3fd2432f7415e2a08fba7ca43ac4d0dc;", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + }, + wantHeader: map[string][]string{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp"}, + "Cookie": {HTTPHeaderRedactedMessage}, + "User-Agent": {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"}, + }, + }, + { + name: "withCookie-not-sanitize", + sanitizeHTTPHeaders: false, + reqMethod: "GET", + reqURL: "https://vtctld-dev-xyz.company.com/api/workflow/poll/11", + reqHeader: map[string]string{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp", + "Cookie": "_ga=GA1.2.1234567899.1234567890; machine-cookie=username:123456789:1234f999ff99aa1af3ae9999d2a5d3968ac014a0947ae1418b10642ab2cbb7d3; session=7e5dc76807be39dd0886b9425883dabe3fd2432f7415e2a08fba7ca43ac4d0dc;", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + }, + wantHeader: map[string][]string{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp"}, + "Cookie": {"_ga=GA1.2.1234567899.1234567890; machine-cookie=username:123456789:1234f999ff99aa1af3ae9999d2a5d3968ac014a0947ae1418b10642ab2cbb7d3; session=7e5dc76807be39dd0886b9425883dabe3fd2432f7415e2a08fba7ca43ac4d0dc;"}, + "User-Agent": {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"}, + }, + }, + { + name: "noCookie-sanitize", + sanitizeHTTPHeaders: true, + reqMethod: "GET", + reqURL: "https://vtctld-dev-xyz.company.com/api/workflow/poll/11", + reqHeader: map[string]string{ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp", + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36", + }, + wantHeader: map[string][]string{ + "Accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp"}, + "User-Agent": {"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36"}, + }, + }, + } + + for _, tc := range testCases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + req, err := http.NewRequestWithContext(ctx, tc.reqMethod, tc.reqURL, nil) + if err != nil { + t.Fatalf("can't parse test http req: %v", err) + } + + for k, v := range tc.reqHeader { + req.Header.Set(k, v) + } + clone := req.Clone(ctx) + + if got := sanitizeRequestHeader(req, tc.sanitizeHTTPHeaders); !reflect.DeepEqual(got.Header, tc.wantHeader) { + t.Errorf("sanitizeRequestHeader() failed\nwant: %#v\ngot: %#v", tc.wantHeader, got.Header) + } + + if !reflect.DeepEqual(clone, req) { + t.Errorf("sanitizeRequestHeader() corrupted original http request\nwant: %#v\ngot: %#v", clone, req) + } + }) + } +} diff --git a/go/vt/workflow/manager.go b/go/vt/workflow/manager.go index 345b51b350a..bf15129a427 100644 --- a/go/vt/workflow/manager.go +++ b/go/vt/workflow/manager.go @@ -17,14 +17,13 @@ limitations under the License. package workflow import ( + "context" "encoding/json" "fmt" "net/url" "sync" "time" - "context" - gouuid "github.com/pborman/uuid" "vitess.io/vitess/go/vt/log" @@ -95,6 +94,8 @@ type Manager struct { started chan struct{} // workflows is a map from job UUID to runningWorkflow. workflows map[string]*runningWorkflow + // sanitizeHTTPHeaders toggles sanitizeRequestHeader() behavior + sanitizeHTTPHeaders bool } // runningWorkflow holds information about a running workflow. @@ -134,6 +135,11 @@ func NewManager(ts *topo.Server) *Manager { } } +// SetSanitizeHTTPHeaders - toggles m.sanitizeHTTPHeaders on/off +func (m *Manager) SetSanitizeHTTPHeaders(to bool) { + m.sanitizeHTTPHeaders = to +} + // SetRedirectFunc sets the redirect function to use. func (m *Manager) SetRedirectFunc(rf func() (string, error)) { m.redirectFunc = rf diff --git a/test/templates/cluster_endtoend_test.tpl b/test/templates/cluster_endtoend_test.tpl index f053bf0fc6e..eab0a143dec 100644 --- a/test/templates/cluster_endtoend_test.tpl +++ b/test/templates/cluster_endtoend_test.tpl @@ -15,6 +15,9 @@ jobs: - name: Tune the OS run: | echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf # TEMPORARY WHILE GITHUB FIXES THIS https://github.com/actions/virtual-environments/issues/3185 - name: Add the current IP address, long hostname and short hostname record to /etc/hosts file @@ -27,8 +30,20 @@ jobs: - name: Get dependencies run: | + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + {{if .Ubuntu20}} + + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + + {{end}} + sudo apt-get update - sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils sudo service mysql stop sudo service etcd stop sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ diff --git a/test/templates/unit_test.tpl b/test/templates/unit_test.tpl index 834a2a51dc8..cbf7ab33109 100644 --- a/test/templates/unit_test.tpl +++ b/test/templates/unit_test.tpl @@ -60,6 +60,8 @@ jobs: {{end}} {{if (eq .Platform "mysql80")}} + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 # mysql80 wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.14-1_all.deb