diff --git a/.github/workflows/build_arm_wheels.yml b/.github/workflows/build_arm_wheels.yml index 7bd8073dc5e..5c14b0aaa5f 100644 --- a/.github/workflows/build_arm_wheels.yml +++ b/.github/workflows/build_arm_wheels.yml @@ -25,7 +25,7 @@ jobs: os: [ ubuntu-20.04 ] env: RUNNER_OS: ${{ matrix.os }} - PYTHON_VERSIONS: "3.8 3.9 3.10 3.11" + PYTHON_VERSIONS: "3.8 3.9 3.10 3.11 3.12" steps: - name: Install clang++ for Ubuntu if: matrix.os == 'ubuntu-20.04' diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index fa9075847a8..25a021bcd89 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -19,7 +19,7 @@ jobs: fail-fast: false matrix: os: [ ubuntu-20.04 ] - python-version: [ "3.8", "3.9", "3.10", "3.11"] + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] # python-version: [ "3.7" ] env: RUNNER_OS: ${{ matrix.os }} @@ -87,7 +87,7 @@ jobs: clang++ --version - name: Run chdb/build.sh run: | - python3 -m pip install pybind11 + python3 -m pip install pybind11 setuptools export CC=/usr/bin/clang export CXX=/usr/bin/clang++ bash ./chdb/build.sh @@ -149,7 +149,7 @@ jobs: matrix: os: [ macos-12 ] # python-version: [ "3.8", "3.9", "3.10", "3.11"] - python-version: [ "3.11" ] + python-version: [ "3.11", "3.12" ] env: RUNNER_OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} @@ -207,7 +207,7 @@ jobs: - name: Run chdb/build.sh timeout-minutes: 300 run: | - python3 -m pip install pybind11 + python3 -m pip install pybind11 setuptools export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin export CC=$(brew --prefix llvm@16)/bin/clang export CXX=$(brew --prefix llvm@16)/bin/clang++ @@ -266,20 +266,20 @@ jobs: TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} build_wheels_macos: - name: cbw on ${{ matrix.os }} + name: ${{ matrix.os }} py${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ macos-11 ] # python-version: [ "3.8", "3.9", "3.10"] - python-version: [ "3.11" ] + python-version: [ "3.8", "3.9", "3.10" ] env: RUNNER_OS: ${{ matrix.os }} PYTHON_VERSION: ${{ matrix.python-version }} steps: - - name: Install Offical Python 3.11 - uses: actions/setup-python@v2 + - name: Install Offical Python + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Check Python version @@ -287,7 +287,7 @@ jobs: python3 -VV python3-config --includes - name: Install clang++ for macOS - if: matrix.os == 'macos-11' || matrix.os == 'macos-12' + if: matrix.os == 'macos-11' run: | pwd uname -a @@ -322,12 +322,17 @@ jobs: key: ${{ matrix.os }} max-size: 5G append-timestamp: true - - name: Prepare chdb/build.sh + - name: Run chdb/build.sh + timeout-minutes: 300 run: | python3 -m pip install pybind11 export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin export CC=$(brew --prefix llvm@16)/bin/clang - export CXX=$(brew --prefix llvm@16)/bin/clang++ + export CXX=$(brew --prefix llvm@16)/bin/clang++ + bash gen_manifest.sh + bash ./chdb/build.sh + python3 -m pip install pandas pyarrow + bash -x ./chdb/test_smoke.sh continue-on-error: false - name: Check ccache statistics run: | @@ -336,24 +341,6 @@ jobs: df -h env: CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++" - # - name: Remove /usr/local/bin/python3 - # run: | - # sudo rm -f /usr/local/bin/python3 - - name: Install dependencies for building wheels - run: | - python3 -m pip install -U pip tox pybind11 twine setuptools wheel - python3 -m pip install cibuildwheel==2.12.1 - - name: Build wheels - run: python3 -m cibuildwheel --output-dir wheelhouse - timeout-minutes: 300 - env: - CIBW_ENVIRONMENT_MACOS: "PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin CC=$(brew --prefix llvm@16)/bin/clang CXX=$(brew --prefix llvm@16)/bin/clang++" - CIBW_DEBUG: 1 - CIBW_BEFORE_BUILD: "pip install -U pip tox pybind11 && bash -x gen_manifest.sh && bash chdb/build.sh" - CIBW_BUILD_VERBOSITY: 3 - CIBW_BUILD: "cp38-macosx_x86_64 cp39-macosx_x86_64 cp310-macosx_x86_64" - CIBW_TEST_REQUIRES: "pyarrow pandas psutil" - CIBW_TEST_COMMAND: "cd {project} && make test" - name: Keep killall ccache and wait for ccache to finish if: always() run: | @@ -362,17 +349,37 @@ jobs: killall ccache; \ sleep 10; \ done + - name: Install dependencies for building wheels + run: | + python3 -m pip install -U pip tox pybind11 twine setuptools wheel + - name: Build wheels + timeout-minutes: 300 + run: | + export PATH=$(brew --prefix llvm@16)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin + export CC=$(brew --prefix llvm@16)/bin/clang + export CXX=$(brew --prefix llvm@16)/bin/clang++ + make wheel + - name: Fix wheel platform tag + run: | + python3 -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl + - name: Run tests + run: | + python3 -m pip install dist/*.whl + python3 -m pip install pandas pyarrow psutil + python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)" + make test + continue-on-error: false - name: Show files - run: ls -lh wheelhouse + run: ls -lh dist shell: bash - uses: actions/upload-artifact@v3 with: - path: ./wheelhouse/*.whl + path: ./dist/*.whl - name: Upload pypi if: startsWith(github.ref, 'refs/tags/v') run: | python3 -m pip install twine - python3 -m twine upload wheelhouse/*.whl + python3 -m twine upload dist/*.whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/chdb/build_linux_arm64.sh b/chdb/build_linux_arm64.sh index 410249ceb81..f6c1ece4bf2 100755 --- a/chdb/build_linux_arm64.sh +++ b/chdb/build_linux_arm64.sh @@ -10,7 +10,7 @@ for PY_VER in ${PYTHON_VERSIONS}; do cd ${PROJ_DIR} pyenv local "${PY_VER}" python3 --version - python3 -m pip install pybind11 + python3 -m pip install pybind11 setuptools export CC=/usr/bin/clang export CXX=/usr/bin/clang++ # Install universal2 pkg diff --git a/chdb/build_mac_arm64.sh b/chdb/build_mac_arm64.sh index 9d00a100fd5..3188d1c428c 100755 --- a/chdb/build_mac_arm64.sh +++ b/chdb/build_mac_arm64.sh @@ -51,7 +51,7 @@ if [ "${dest}" != "darwin-x86_64" ]; then fi -for PY_VER in 3.9.13 3.10.11 3.11.3; do +for PY_VER in 3.9.13 3.10.11 3.11.3 3.12.0; do if [ ! -f ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg ]; then wget https://www.python.org/ftp/python/${PY_VER}/python-${PY_VER}-macos11.pkg -O ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg fi @@ -73,7 +73,7 @@ for PY_VER in 3.9.13 3.10.11 3.11.3; do exit 1 fi - python3 -m pip install -U pybind11 wheel build tox psutil + python3 -m pip install -U pybind11 wheel build tox psutil setuptools rm -rf ${PROJ_DIR}/buildlib ${PROJ_DIR}/chdb/build.sh diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index baaf455b7bf..e8ba7a8ee65 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -58,6 +58,7 @@ #include #include #include +#include #include "config.h" @@ -801,6 +802,42 @@ void LocalServer::processConfig() global_context->getUserDefinedSQLObjectsLoader().loadObjects(); LOG_DEBUG(log, "Loaded metadata."); + + /** Set default database if it is specified in default_database file. + * NOTE: We do it after loading metadata to let the '_local' and 'system' database initialization + * to be done correctly. + * I have tried to set default database during parsing '--database' option, but it leads to + * "Code: 82. DB::Exception: Database db_xxx already exists.: while loading database `db_xxx` + * from path .state_tmp_auxten_usedb_/metadata/db_xxx. (DATABASE_ALREADY_EXISTS)" + * This will also happen if we call: + * `clickhouse local --database=db_xxx --path=.state_tmp_auxten_usedb_ --query="select * FROM log_table_xxx"` + * with existing `db_xxx` database in the '.state_tmp_auxten_usedb_' directory. + */ + auto default_database_path = fs::path(path) / "default_database"; + if (std::filesystem::exists(default_database_path)) + { + std::ifstream ifs(default_database_path); + std::string user_default_database; + if (ifs.is_open()) + { + ifs >> user_default_database; + // strip default_database + user_default_database.erase( + std::remove_if( + user_default_database.begin(), user_default_database.end(), [](unsigned char x) { return std::isspace(x); }), + user_default_database.end()); + if (!user_default_database.empty()) + { + global_context->setCurrentDatabase(user_default_database); + LOG_DEBUG(log, "Set default database to {} recorded in {}", user_default_database, default_database_path); + } + ifs.close(); + } + else + { + LOG_ERROR(log, "Cannot read default database from {}", default_database_path); + } + } } else if (!config().has("no-system-tables")) { diff --git a/src/Interpreters/InterpreterUseQuery.cpp b/src/Interpreters/InterpreterUseQuery.cpp index b71f3a9cc1c..f8f413332fa 100644 --- a/src/Interpreters/InterpreterUseQuery.cpp +++ b/src/Interpreters/InterpreterUseQuery.cpp @@ -2,17 +2,40 @@ #include #include #include +#include #include +#include namespace DB { +namespace ErrorCodes +{ + extern const int CANNOT_OPEN_FILE; +} + BlockIO InterpreterUseQuery::execute() { const String & new_database = query_ptr->as().getDatabase(); getContext()->checkAccess(AccessType::SHOW_DATABASES, new_database); getContext()->getSessionContext()->setCurrentDatabase(new_database); + + // Save the current using database in default_database stored in getPath() + // for the case when the database is changed in chDB session. + // The default_database content is used in the LocalServer::processConfig() method. + auto default_database_path = fs::path(getContext()->getPath()) / "default_database"; + std::ofstream tmp_path_fs(default_database_path, std::ofstream::out | std::ofstream::trunc); + if (tmp_path_fs && tmp_path_fs.is_open()) + { + tmp_path_fs << new_database; + tmp_path_fs.close(); + } + else + { + throw Exception(ErrorCodes::CANNOT_OPEN_FILE, "Cannot open file {} for writing", default_database_path.string()); + } + return {}; } diff --git a/tests/test_usedb.py b/tests/test_usedb.py new file mode 100644 index 00000000000..0eed3e09589 --- /dev/null +++ b/tests/test_usedb.py @@ -0,0 +1,42 @@ +#!python3 + +import unittest +import shutil +import psutil +from chdb import session + + +test_state_dir = ".state_tmp_auxten_usedb_" +current_process = psutil.Process() +check_thread_count = False + + +class TestStateful(unittest.TestCase): + def setUp(self) -> None: + shutil.rmtree(test_state_dir, ignore_errors=True) + return super().setUp() + + def tearDown(self) -> None: + shutil.rmtree(test_state_dir, ignore_errors=True) + return super().tearDown() + + def test_path(self): + sess = session.Session(test_state_dir) + + sess.query("CREATE DATABASE IF NOT EXISTS db_xxx ENGINE = Atomic", "CSV") + ret = sess.query("SHOW DATABASES", "CSV") + self.assertIn("db_xxx", str(ret)) + + sess.query("CREATE TABLE IF NOT EXISTS db_xxx.log_table_xxx (x UInt8) ENGINE = Log;") + sess.query("INSERT INTO db_xxx.log_table_xxx VALUES (1), (2), (3), (4);") + + ret = sess.query("USE db_xxx; SELECT * FROM log_table_xxx", "Debug") + self.assertEqual(str(ret), "1\n2\n3\n4\n") + + sess.query("USE db_xxx") + ret = sess.query("SELECT * FROM log_table_xxx", "Debug") + self.assertEqual(str(ret), "1\n2\n3\n4\n") + + +if __name__ == '__main__': + unittest.main()