From dd967b635ab99b476a7395cbc9c817c6df51ca72 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Tue, 9 Jun 2026 18:01:36 +0200 Subject: [PATCH] Fix development/ scripts to work around various issues - Makefile proxy: database name has changed - dump-schema.sh: check for mysqldump, check whether it is mariadbdump, and if either not available or mariadb, use docker to run the code - db_codegen.py: don't hard-code venv/bin/python, it conflicts with pyenv - db_codegen.py: detect mariadb/mysql client and use the respective ssl flags --- development/Makefile | 4 ++-- development/db_codegen.py | 26 ++++++++++++++++++++++++-- development/dump-schema.sh | 28 +++++++++++++++++++++++++--- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/development/Makefile b/development/Makefile index e23caa73e..f78319756 100644 --- a/development/Makefile +++ b/development/Makefile @@ -10,8 +10,8 @@ hello: proxy: - /usr/local/bin/cloud-sql-proxy --address 0.0.0.0 --port ${PROD_DB_PROXY_PORT} arxiv-production:us-central1:arxiv-production-rep9 > /dev/null 2>&1 & + cloud-sql-proxy --address 0.0.0.0 --port ${PROD_DB_PROXY_PORT} arxiv-production:us-central1:arxiv-production-rep11 > /dev/null 2>&1 & dev-proxy: - /usr/local/bin/cloud-sql-proxy --address 0.0.0.0 --port ${DEV_DB_PROXY_PORT} arxiv-development:us-east4:arxiv-db-dev > /dev/null 2>&1 & + cloud-sql-proxy --address 0.0.0.0 --port ${DEV_DB_PROXY_PORT} arxiv-development:us-east4:arxiv-db-dev > /dev/null 2>&1 & diff --git a/development/db_codegen.py b/development/db_codegen.py index f45f2afcd..4c3fefb83 100644 --- a/development/db_codegen.py +++ b/development/db_codegen.py @@ -523,6 +523,28 @@ def run_mysql_container(port: int): logging.error(f"Unexpected error: {e}") +def is_mariadb_client(): + """Detect whether the `mysql` binary is actually the MariaDB client. + + MariaDB ships a `mysql` compatibility wrapper that does not understand + MySQL's `--ssl-mode` option (it uses `--ssl`/`--skip-ssl` instead). + """ + try: + result = subprocess.run( + ["mysql", "--version"], capture_output=True, text=True, check=True + ) + return "mariadb" in result.stdout.lower() + except (subprocess.CalledProcessError, FileNotFoundError): + return False + + +def ssl_client_flag(ssl_enabled: bool): + """Return the client SSL flag appropriate for the installed `mysql` client.""" + if is_mariadb_client(): + return "--ssl" if ssl_enabled else "--skip-ssl" + return "--ssl-mode=REQUIRED" if ssl_enabled else "--ssl-mode=disabled" + + def load_sql_file(sql_file, mysql_port=3306, ssl_enabled=True): """Load a sql file to mysql @@ -542,7 +564,7 @@ def load_sql_file(sql_file, mysql_port=3306, ssl_enabled=True): f"--port={mysql_port}", "-uroot", "-ptestpassword", - "--ssl-mode=REQUIRED" if ssl_enabled else "--ssl-mode=disabled", + ssl_client_flag(ssl_enabled), "testdb" ], stdin=sql, check=True) logging.info(f"SQL file '{sql_file}' loaded successfully into 'testdb' on port {mysql_port}.") @@ -606,7 +628,7 @@ def main() -> None: if env.get("PYTHONPATH"): pythonpath = pythonpath + os.pathsep + env["PYTHONPATH"] env["PYTHONPATH"] = pythonpath - subprocess.run(['venv/bin/python', '-m', 'sqlacodegen', p_url, '--outfile', p_outfile, + subprocess.run(['python', '-m', 'sqlacodegen', p_url, '--outfile', p_outfile, '--model-metadata', db_metadata], check=True, cwd=arxiv_base_dir, env=env) # autogen_models.py diff --git a/development/dump-schema.sh b/development/dump-schema.sh index cbf4796cd..e7ac0fe72 100755 --- a/development/dump-schema.sh +++ b/development/dump-schema.sh @@ -1,4 +1,26 @@ #!/bin/bash -# -PASSWORD=$(< ~/.arxiv/arxvi-db-read-only-password) -mysqldump -h 127.0.0.1 --port 2021 -u readonly -p"$PASSWORD" --no-data --set-gtid-purged=OFF --skip-comments arXiv | sed 's/ AUTO_INCREMENT=[0-9]*\b/ AUTO_INCREMENT=1/' > arxiv/db/arxiv_db_schema.sql + + +PASSWORD=$(gcloud secrets versions access latest --project=arxiv-production --secret=arxiv-production-rep11-db-readonly) + +USE_DOCKER=0 +if command -v mysqldump &>/dev/null +then + # mysqldump is here, but could be mariadump which does NOT WORK! + if mysqldump --version 2>&1 | grep -qi mariadb ; then + USE_DOCKER=1 + fi +else + USE_DOCKER=1 +fi + +DOCKER="" +if [ $USE_DOCKER = 1 ] ; then + DOCKER="docker run --net=host -ti mysql:8.4.5 " +fi +$DOCKER mysqldump -h 127.0.0.1 --port 2021 -u readonly -p"$PASSWORD" --no-data --set-gtid-purged=OFF --skip-comments arXiv | sed 's/ AUTO_INCREMENT=[0-9]*\b/ AUTO_INCREMENT=1/' > arxiv/db/arxiv_db_schema.sql + + +echo "CHECK FOR LINE ENDINGS, MIGHT BE DOS!!" +echo "CHECK FOR PASSWORD WARNING ON TOP OF arxiv/db/arxiv_db_schema.sql" +