diff --git a/.github/actions/fuse_compat/action.yml b/.github/actions/fuse_compat/action.yml index 76babaaa9de0d..c2c5a5c1d2fab 100644 --- a/.github/actions/fuse_compat/action.yml +++ b/.github/actions/fuse_compat/action.yml @@ -19,25 +19,15 @@ runs: sha: ${{ github.sha }} target: ${{ inputs.target }} path: ./bins/current - + - name: Setup Build Tool + uses: ./.github/actions/setup_build_tool + with: + bypass_env_vars: RUSTFLAGS,RUSTDOCFLAGS,RUST_TEST_THREADS,RUST_LOG,RUST_BACKTRACE - name: Test compatibility shell: bash run: | - docker run --rm --tty --net=host \ - --user $(id -u):$(id -g) \ - --env BUILD_PROFILE \ - --volume "${PWD}:/workspace" \ - --workdir "/workspace" \ - datafuselabs/build-tool:sqllogic \ bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.150 - docker run --rm --tty --net=host \ - --user $(id -u):$(id -g) \ - --env BUILD_PROFILE \ - --volume "${PWD}:/workspace" \ - --workdir "/workspace" \ - datafuselabs/build-tool:sqllogic \ bash ./tests/fuse-compat/test-fuse-compat.sh 0.7.151 - - name: Upload failure if: failure() uses: ./.github/actions/artifact_failure diff --git a/.github/workflows/databend-release.yml b/.github/workflows/databend-release.yml index c1bb873993747..728077e29f188 100644 --- a/.github/workflows/databend-release.yml +++ b/.github/workflows/databend-release.yml @@ -308,7 +308,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | version=${{ needs.create_release.outputs.version }} - tar -C ./tests/logictest -czvf testsuites-${version}.tar.gz suites + tar -C ./tests/sqllogictests -czvf testsuites-${version}.tar.gz suites gh release upload ${version} testsuites-${version}.tar.gz --clobber release_docker_combined: diff --git a/tests/compat/test-compat.sh b/tests/compat/test-compat.sh index e3b40417bddae..009ed244f3524 100755 --- a/tests/compat/test-compat.sh +++ b/tests/compat/test-compat.sh @@ -208,10 +208,8 @@ run_test() { echo " === Run metasrv related test: 05_ddl" if [ "$query_ver" = "current" ]; then - cd "$SCRIPT_PATH/../../tests/logictest" || exit # Only run test on mysql handler - python3 main.py "_ddl_" --handlers mysql - cd - + cargo run -p sqllogictests -- --handlers mysql --run_dir 05_ddl else ( # download suites into ./old_suite diff --git a/tests/fuse-compat/test-fuse-compat.sh b/tests/fuse-compat/test-fuse-compat.sh index 14db79ecb4f59..06ee943a7d652 100755 --- a/tests/fuse-compat/test-fuse-compat.sh +++ b/tests/fuse-compat/test-fuse-compat.sh @@ -90,11 +90,9 @@ git_partial_clone() { # Run specified tests found in logic test suite dir run_logictest() { - local pattern="$1" ( - cd "tests/logictest" # Only run test on mysql handler - python3 main.py "$pattern" --handlers mysql --suites "$SCRIPT_PATH/compat-logictest" + cargo run -p sqllogictests -- --handlers mysql --suites "$logictest_path" ) } diff --git a/tests/logictest/Dockerfile b/tests/logictest/Dockerfile deleted file mode 100644 index df0b60615c23f..0000000000000 --- a/tests/logictest/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.10 - -RUN mkdir /test -WORKDIR /test - -COPY *.py /test -COPY requirements.txt /test - -RUN apt-get update -yq && apt-get install -yq \ - psmisc libffi-dev && \ - rm -rf /var/lib/apt/lists/* - -RUN python3 -m pip install --upgrade pip && pip install -r requirements.txt - -VOLUME ["/test/suites"] -CMD ["/test/main.py"] diff --git a/tests/logictest/README.md b/tests/logictest/README.md deleted file mode 100644 index 8344d48053cbf..0000000000000 --- a/tests/logictest/README.md +++ /dev/null @@ -1,213 +0,0 @@ -# Sqllogic test - -The database return right with different handlers, for example MySQL and http - -# Usage - -## Prepare - -Change to the scripts dir: - -```shell -cd tests/logictest/ -``` - -Make sure python3 is installed. - -If you are familiar with `pip`, you can install dependency with: - -```shell -pip install -r requirements.txt -``` - -## Need to know - -1. Cases from **tests/suites/0_stateless/** to **tests/logictest/suites/gen/** -2. If a case file already exists in gen/, gen_suites will ignore it. -3. Regenerate：delete case file in gen/ and run gen_suites.py - -## Generate sqllogic test cases from Stateless Test - -1. python3 gen_suites.py - -## Usage - -You can simply run all tests with: - -```shell -python main.py -``` - -Get help with: - -```shell -python main.py -h -``` - -Useful arguments: - -1. --run-dir ydb will only run the suites in dir ./suites/ydb/ -2. --skip-dir ydb will skip the suites in dir ./suites/ydb -3. --suites other_dir wiil use suites file in dir ./other_dir -4. Run files by pattern string like: python main.py "03_0001" - -## Docker - -### Build image - -docker build -t sqllogic/test:latest . - -### Run with docker - -1. Image release: datafuselabs/sqllogictest:latest -2. Set envs - -- SKIP_TEST_FILES (skip test case, set file name here split by `,` ) -- QUERY_MYSQL_HANDLER_HOST -- QUERY_MYSQL_HANDLER_PORT -- QUERY_HTTP_HANDLER_HOST -- QUERY_HTTP_HANDLER_PORT -- MYSQL_DATABASE -- MYSQL_USER -- ADDITIONAL_HEADERS (for security scenario) - -3. docker run --name logictest --rm --network host datafuselabs/sqllogictest:latest - -## How to write logic test - -Fast start, you can follow this demo: https://github.com/datafuselabs/databend/blob/main/tests/logictest/suites/select_0 - -Runner supported: mysql handler, http handler, clickhouse handler. - -- ok - - Returns no error, don't care about the result -- error - - Returns with error and expected error message, usually with an error code, but also with a message string; the way to determine whether the specified string is in the returned message -- query - - Return result and check the result with expected, follow by query_type and query_label - - query_type is a char represent a column in result, multi char means multi column - - B Boolean - - T text - - F floating point - - I integer - - R regex - - query_label If different runner return inconsistency, you can write like this(suppose that mysql handler is get different result) - -This is a query demo(query_label is optional): - -``` -query III label(mysql) -select number, number + 1, number + 999 from numbers(10); ----- - 0 1 999 - 1 2 1000 - 2 3 1001 - 3 4 1002 - 4 5 1003 - 5 6 1004 - 6 7 1005 - 7 8 1006 - 8 9 1007 - 9 10 1008.0 - ----- mysql - 0 1 999 - 1 2 1000 - 2 3 1001 - 3 4 1002 - 4 5 1003 - 5 6 1004 - 6 7 1005 - 7 8 1006 - 8 9 1007 - 9 10 1008 -``` - -## How to use regex in logic test - -1. Regular expressions are implemented in regex_type.py. Additions can be made as needed but modifications require particular care. - -``` -regex_type_map = { - "ANYTHING": ".*", - "DATE": "\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d[.]\d\d\d [+-]\d\d\d\d", -} -``` - -2. A demo of regex likes: - -``` -query TTTRRTIIII -SELECT * FROM system.tables WHERE database='db1'; ----- -default db1 t1 FUSE $ANYTHING $DATE NULL 0 0 0 0 -``` - -### limitation - -The space is used to split results into columns, space in regex expression should specify the number, do not use `( )+` or `( )*`. Do not contain space in the expected result of regex expression: ANYTHING, it should be written as a single column. - -## Write logic test tips - -1. skipif help you skip test of given handler - -``` -skipif clickhouse -query I -select 1; ----- -1 -``` - -2. onlyif help you run test only by given handler - -``` -onlyif mysql -query I -select 1; ----- -1 -``` - -3. if some test has a flaky failure, and you want to ignore it, simply add skipped before statement query. (Remove it after the problem is solved) - -``` -query skipped I -select 1; ----- -1 -``` - -### Acknowledgement - -- _tips_ If you do not care about results, use statement ok instead of statement query -- _tips_ Add ORDER BY to ensure that the order of returned results is always consistent -- _warning_ A statement query need results, and even if you want to skip a case, you still need to keep the results in the test content - -## Tools - -complete.py can auto-complete test file for you, It does as follow steps: - -1. Get SQLs from source-file, whether an SQL file or logictest file. -2. Execute SQL one by one, if SQL fetches results, add query; if SQL fetches nothing, add statement ok; if SQL gets an error, add statement error. - -### Usage - -- Pre-run, you need to start databend server or MySQL server -- Use `./complete.py --source-file="xxx.sql" --dest-file="my-gen"` for SQL files(suffix name must be like \*.sql) -- Use `./complete.py --source-file="xxx.test" --fest-file="my-gen"` for logictest files(suffix name not like _.sql, maybe like _.test) -- Use `--enable-auto-cleanup` to add `drop table if exists xxx` or `drop database if exists xxx` at the beginning of the test file -- If you want to see what SQLs get from source-file, add `--show-sql` -- Use the command line to specify host, user, port, password and database. Details in `./complete.py -h` - -### Acknowledgement - -- _tips_ You can use MYSQL syntax to auto-complete the test suite, but make sure you know all grammar differences. -- _tips_ MYSQL return bool as 1 and 0, this tool makes it as `int(I)` in query type option. -- _warning_ No multi handlers use in the auto-complete tool(MYSQL only), if handlers return a difference, manual fix it, please. - -# Learn More - -RFC: https://github.com/datafuselabs/databend/blob/main/docs/doc/60-contributing/03-rfcs/20220425-new_sql_logic_test_framework.md -Migration discussion: https://github.com/datafuselabs/databend/discussions/5838 - diff --git a/tests/logictest/cleanup.py b/tests/logictest/cleanup.py deleted file mode 100644 index 78fbc00dce571..0000000000000 --- a/tests/logictest/cleanup.py +++ /dev/null @@ -1,51 +0,0 @@ -# tuple of (type, name) -# example ('database', 'db1'), ('table', 't1') -need_cleanup_set = set() -enable_auto_cleanup = False - - -def set_auto_cleanup(flag): - global enable_auto_cleanup - enable_auto_cleanup = flag - - -def get_cleanup_statements(): - if not enable_auto_cleanup: - return [] - global need_cleanup_set - res = [] - for type, name in need_cleanup_set: - res.append(f"drop {type} if exists {name};") - need_cleanup_set = set() - return res - - -def pick_create_statement(statement): - if not enable_auto_cleanup: - return - global need_cleanup_set - statement_lower = statement.lower() - if "create" not in statement_lower: - return - statement_words = statement_lower.strip(";").split() - - create_type_index = 1 - create_name_index = 2 - if ( - "if" in statement_lower - and "not" in statement_lower - and "exists" in statement_lower - ): - create_type_index = 5 - elif "if" in statement_lower and "exists" in statement_lower: - create_type_index = 4 - else: - create_type_index = 2 - - if "transient" in statement_lower: - create_type_index += 1 - create_name_index += 1 - - create_type = statement_words[1] - create_name = statement_words[create_name_index] - need_cleanup_set.add((create_type, create_name.split("(")[0])) diff --git a/tests/logictest/clickhouse_connector.py b/tests/logictest/clickhouse_connector.py deleted file mode 100644 index ebaa77dfc1344..0000000000000 --- a/tests/logictest/clickhouse_connector.py +++ /dev/null @@ -1,58 +0,0 @@ -import environs -import os - -from log import log - -default_database = "default" - - -class ClickhouseConnector(object): - def connect(self, host, port, user="root", password="", database=default_database): - - protocol = os.getenv("QUERY_CLICKHOUSE_HANDLER_PROTOCAL") - if protocol is None: - if port == "443" or port == "8443": - protocol = "https" - else: - protocol = "http" - self._uri = f"clickhouse+http://{user}:{password}@{host}:{port}/{database}?protocol={protocol}" - log.debug(self._uri) - e = environs.Env() - self._additonal_headers = dict() - if os.getenv("CLICKHOUSE_ADDITIONAL_HEADERS") is not None: - headers = e.dict("CLICKHOUSE_ADDITIONAL_HEADERS") - for key in headers: - self._additonal_headers["header__" + key] = headers[key] - - self._session = None - - def query_with_session(self, statement): - from clickhouse_sqlalchemy import make_session # type: ignore - from sqlalchemy import create_engine # type: ignore - - if self._session is None: - engine = create_engine(self._uri, connect_args=self._additonal_headers) - self._session = make_session(engine) - log.debug(statement) - return self._session.execute(statement) - - def reset_session(self): - if self._session is not None: - self._session.close() - self._session = None - - def fetch_all(self, statement): - cursor = self.query_with_session(statement) - data_list = list() - for item in cursor.fetchall(): - data_list.append(list(item)) - cursor.close() - return data_list - - -# if __name__ == '__main__': -# from config import clickhouse_config -# connector = ClickhouseConnector() -# connector.connect(**clickhouse_config) -# print(connector.fetch_all("show databases")) -# print(connector.fetch_all("select * from t1")) diff --git a/tests/logictest/clickhouse_runner.py b/tests/logictest/clickhouse_runner.py deleted file mode 100644 index d69eb9517fe9a..0000000000000 --- a/tests/logictest/clickhouse_runner.py +++ /dev/null @@ -1,51 +0,0 @@ -from abc import ABC -from log import log -from mysql.connector.errors import Error - -import logictest -import clickhouse_connector - - -class TestClickhouse(logictest.SuiteRunner, ABC): - def __init__(self, kind, args): - super().__init__(kind, args) - self._ch = None - - def get_connection(self): - if self._ch is None: - self._ch = clickhouse_connector.ClickhouseConnector() - self._ch.connect(**self.driver) - return self._ch - - def reset_connection(self): - if self._ch is not None: - self._ch.reset_session() - - def batch_execute(self, statement_list): - for statement in statement_list: - self.execute_statement(statement) - self.reset_connection() - - def execute_ok(self, statement): - self.get_connection().query_with_session(statement) - return None - - def execute_error(self, statement): - try: - self.get_connection().query_with_session(statement) - except Exception as err: - return Error(msg=str(err)) - - def execute_query(self, statement): - results = self.get_connection().fetch_all(statement.text) - log.debug(results) - # query_type = statement.s_type.query_type - vals = [] - for (ri, row) in enumerate(results): - for (i, v) in enumerate(row): - if isinstance(v, type(None)): - vals.append("NULL") - continue - - vals.append(str(v)) - return vals diff --git a/tests/logictest/complete.py b/tests/logictest/complete.py deleted file mode 100644 index 416554c4ac017..0000000000000 --- a/tests/logictest/complete.py +++ /dev/null @@ -1,234 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -import os -import re - -from argparse import ArgumentParser -import mysql.connector - -from logictest import is_empty_line -from config import mysql_config -from http_connector import format_result -from cleanup import pick_create_statement, get_cleanup_statements, set_auto_cleanup - -error_code_regex = r".*Code: (?P.*),.*" -target_dir = "./" - - -def get_error_code(msg): - matched = re.match(error_code_regex, msg, re.MULTILINE | re.IGNORECASE) - if matched is None: - return "{error_code}" - return matched.group("error_code") - - -def parse_sql_file(source_file): - sqls = list() - f = open(source_file, encoding="UTF-8") - sql_content = "" - skipped_query = False - for line in f.readlines(): - if is_empty_line(line): - continue - - if line.startswith("--"): # pass comment - continue - - # multi line sql - sql_content = sql_content + "\n" + line.rstrip() - if ";" not in line: - continue - - statement = sql_content.strip() - sqls.append([statement, skipped_query]) - sql_content = "" - - f.close() - return sqls - - -def parse_logictest_file(source_file): - parsing_statement = False - skipped_query = False - sqls = list() - f = open(source_file, encoding="UTF-8") - sql_content = "" - - for line in f.readlines(): - if is_empty_line(line): - if parsing_statement and skipped_query: - continue - - if parsing_statement: - parsing_statement = False - statement = sql_content.strip() - if ";" not in statement: - statement = statement + ";" - - sqls.append([statement, skipped_query]) - sql_content = "" - continue - - if line.startswith("----") and parsing_statement: - parsing_statement = False - statement = sql_content.strip() - if ";" not in statement: - statement = statement + ";" - sqls.append([statement, skipped_query]) - sql_content = "" - - if line.startswith("--") or line.startswith("#"): # pass comment - continue - - if line.startswith("statement") or line.startswith("query"): - parsing_statement = True - if line.find("skipped") >= 0: - skipped_query = True - continue - - if parsing_statement: - sql_content = sql_content + "\n" + line.rstrip() - if ";" not in line: - continue - - parsing_statement = False - statement = sql_content.strip() - sqls.append([statement, skipped_query]) - skipped_query = False - sql_content = "" - - f.close() - return sqls - - -def get_sql_from_file(source_file): - if ".sql" in os.path.basename(source_file): - return parse_sql_file(source_file) - else: - return parse_logictest_file(source_file) - - -def gen_suite_from_sql(sql_and_skips, dest_file): - out = open(f"{dest_file}", mode="w+", encoding="UTF-8") - statements = list() - connection = mysql.connector.connect(**mysql_config) - cursor = connection.cursor(buffered=True) - for sql_and_skip in sql_and_skips: - sql = sql_and_skip[0] - if sql_and_skip[1]: - statements.append(f"statement query skipped\n{sql}\n\n") - continue - # use mysql connector - try: - cursor.execute(sql) - pick_create_statement(sql) - except mysql.connector.Error as err: - statements.append(f"statement error {get_error_code(err.msg)}\n{sql}\n\n") - continue - - try: - r = cursor.fetchall() - results = [] - options = "" - for (ri, row) in enumerate(r): - rows = [] - for (i, v) in enumerate(row): - if len(options) <= i: - if isinstance(v, int): - options += "I" - elif isinstance(v, float): - options += "F" - else: - options += "T" - - if isinstance(v, type(None)): - rows.append("NULL") - continue - rows.append(str(v)) - results.append(rows) - if len(results) > 0: - statements.append( - f"statement query {options}\n{sql}\n\n----\n{format_result(results)}\n" - ) - else: - statements.append(f"statement ok\n{sql}\n\n") - except mysql.connector.Error: - statements.append(f"statement ok\n{sql}\n\n") - - # cleanup database, table - drop_statements = list() - for cleanup_sql in get_cleanup_statements(): - try: - drop_statements.append(f"statement ok\n{cleanup_sql}\n\n") - cursor.execute(cleanup_sql) - print(f"Cleanup execute sql: {cleanup_sql}") - except Exception: - pass - - out.writelines(drop_statements) - out.writelines(statements) - out.flush() - out.close() - - -def run(args): - if not os.path.isfile(args.source_file): - print(f"{args.source_file} is not a file") - return - print(f"Source file: {args.source_file}") - print(f"Dest file: {args.dest_file}") - - mysql_config["user"] = args.mysql_user - mysql_config["host"] = args.mysql_host - mysql_config["port"] = args.mysql_port - mysql_config["passwd"] = args.mysql_passwd - mysql_config["database"] = args.mysql_database - - print(f"Mysql config: {mysql_config}") - set_auto_cleanup(args.enable_auto_cleanup) - sql_and_skips = get_sql_from_file(args.source_file) - if args.show_sql: - for sql in sql_and_skips: - print(sql[0]) - - gen_suite_from_sql(sql_and_skips, args.dest_file) - - -if __name__ == "__main__": - parser = ArgumentParser( - description="databend sqllogictest auto-complete tools(from *.sql files or logictest files)" - ) - parser.add_argument("--source-file", help="Path to suites source file") - - parser.add_argument( - "--dest-file", default="./auto", help="Path to logictest auto-complete file" - ) - - parser.add_argument( - "--show-sql", - action="store_true", - default=False, - help="Show sql from source file", - ) - - parser.add_argument( - "--enable-auto-cleanup", - action="store_true", - default=False, - help="Enable auto cleanup after test per session", - ) - - parser.add_argument("--mysql-user", default="root", help="Mysql user") - - parser.add_argument("--mysql-host", default="127.0.0.1", help="Mysql host") - - parser.add_argument("--mysql-port", default="3307", help="Mysql port") - - parser.add_argument("--mysql-passwd", default="root", help="Mysql password") - - parser.add_argument( - "--mysql-database", default="default", help="Mysql default database" - ) - - args = parser.parse_args() - run(args) diff --git a/tests/logictest/config.py b/tests/logictest/config.py deleted file mode 100644 index 34fac8f3ebbf6..0000000000000 --- a/tests/logictest/config.py +++ /dev/null @@ -1,72 +0,0 @@ -import os - -mysql_config = { - "user": "root", - "host": "127.0.0.1", - "port": 3307, - "database": "default", - "raise_on_warnings": True, -} - -http_config = { - "user": "root", - "host": "127.0.0.1", - "port": 8000, - "database": "default", -} - -clickhouse_config = { - "user": "root", - "password": "", - "host": "127.0.0.1", - "port": 8124, - "database": "default", -} - - -def config_from_env(): - mysql_host = os.getenv("QUERY_MYSQL_HANDLER_HOST") - if mysql_host is not None: - mysql_config["host"] = mysql_host - - mysql_port = os.getenv("QUERY_MYSQL_HANDLER_PORT") - if mysql_port is not None: - mysql_config["port"] = int(mysql_port) - - http_host = os.getenv("QUERY_HTTP_HANDLER_HOST") - if http_host is not None: - http_config["host"] = http_host - - http_port = os.getenv("QUERY_HTTP_HANDLER_PORT") - if http_port is not None: - http_config["port"] = int(http_port) - - clickhouse_host = os.getenv("QUERY_CLICKHOUSE_HANDLER_HOST") - if clickhouse_host is not None: - clickhouse_config["host"] = clickhouse_host - - clickhouse_port = os.getenv("QUERY_CLICKHOUSE_HANDLER_PORT") - if clickhouse_port is not None: - clickhouse_config["port"] = clickhouse_port - - clickhouse_password = os.getenv("QUERY_CLICKHOUSE_HANDLER_PASSWORD") - if clickhouse_password is not None: - clickhouse_config["password"] = clickhouse_password - - clickhouse_user = os.getenv("QUERY_CLICKHOUSE_HANDLER_USER") - if clickhouse_user is not None: - clickhouse_config["user"] = clickhouse_user - - mysql_database = os.getenv("MYSQL_DATABASE") - if mysql_database is not None: - mysql_config["database"] = mysql_database - http_config["database"] = mysql_database - clickhouse_config["database"] = mysql_database - - mysql_user = os.getenv("MYSQL_USER") - if mysql_user is not None: - mysql_config["user"] = mysql_user - http_config["user"] = mysql_user - - -config_from_env() diff --git a/tests/logictest/gen_suites.py b/tests/logictest/gen_suites.py deleted file mode 100644 index 861f93ec2d267..0000000000000 --- a/tests/logictest/gen_suites.py +++ /dev/null @@ -1,295 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- -# This is a generator of test cases -# Turn cases in directory ../suites/0_stateless/* into sqllogictest formation -import os -import re -import copy -import time - -import mysql.connector -import fire - -from log import log -from config import mysql_config, http_config -from logictest import is_empty_line -from http_connector import HttpConnector, format_result - -suite_path = "../suites/0_stateless/" -logictest_path = "./suites/gen/" -skip_exist = True - -database_regex = r"use|USE (?P.*);" -error_regex = r"(?P.*)-- {ErrorCode (?P.*)}" -query_statment_first_words = ["select", "show", "explain", "describe"] - -exception_sqls = [] -manual_cases = [] - -STATEMENT_OK = """statement ok -{statement} - -""" - -STATEMENT_ERROR = """statement error {error_id} -{statement} - -""" - -STATEMENT_QUERY = """query {query_options_with_labels} -{statement} - -{results} -""" - -# results_string looks like, result is seperate by space. -# 1 1 1 -RESULTS_TEMPLATE = """{label_separate} -{results_string}""" - - -def first_word(text): - return text.split()[0] - - -def get_error_statment(line): - return re.match(error_regex, line, re.MULTILINE | re.IGNORECASE) - - -def get_database(line): - return re.match(database_regex, line, re.MULTILINE | re.IGNORECASE) - - -# get_all_cases read all file from suites dir -# but only parse .sql file, .result file will be ignore, run sql and fetch results -# need a local running databend-meta and databend-query or change config.py to your cluster -def get_all_cases(): - # copy from databend-test - def collect_subdirs_with_pattern(cur_dir_path, pattern): - return list( - # Make sure all sub-dir name starts with [0-9]+_*. - filter( - lambda fullpath: os.path.isdir(fullpath) - and re.search(pattern, fullpath.split("/")[-1]), - map( - lambda _dir: os.path.join(cur_dir_path, _dir), - os.listdir(cur_dir_path), - ), - ) - ) - - def collect_files_with_pattern(cur_dir_path, patterns): - return list( - filter( - lambda fullpath: os.path.isfile(fullpath) - and os.path.splitext(fullpath)[1] in patterns.split("|"), - map( - lambda _dir: os.path.join(cur_dir_path, _dir), - os.listdir(cur_dir_path), - ), - ) - ) - - def get_all_tests_under_dir_recursive(suite_dir): - all_tests = copy.deepcopy(collect_files_with_pattern(suite_dir, ".sql|.sh|.py")) - # Collect files in depth 0 directory. - sub_dir_paths = copy.deepcopy( - collect_subdirs_with_pattern(suite_dir, "^[0-9]+") - ) - # Recursively get files from sub-directories. - while len(sub_dir_paths) > 0: - cur_sub_dir_path = sub_dir_paths.pop(0) - - all_tests += copy.deepcopy( - collect_files_with_pattern(cur_sub_dir_path, ".sql|.sh|.py") - ) - - sub_dir_paths += copy.deepcopy( - collect_subdirs_with_pattern(cur_sub_dir_path, "^[0-9]+") - ) - return all_tests - - return get_all_tests_under_dir_recursive(suite_path) - - -def parse_cases(sql_file): - # New session every case file - http_client = HttpConnector(**http_config) - cnx = mysql.connector.connect(**mysql_config) - mysql_client = cnx.cursor() - - def mysql_fetch_results(sql): - ret = "" - try: - mysql_client.execute(sql) - r = mysql_client.fetchall() - - for row in r: - rowlist = [] - for item in row: - if item is None: - item = "NULL" - rowlist.append(str(item)) - row_string = " ".join(rowlist) - if len(row_string) == 0: # empty line replace with tab - row_string = "\t" - ret = ret + row_string + "\n" - except Exception as err: - log.warning( - f"SQL: {sql} fetch no results, msg:{str(err)} ,check it manual." - ) - return ret - - target_dir = os.path.dirname(str.replace(sql_file, suite_path, logictest_path)) - case_name = os.path.splitext(os.path.basename(sql_file))[0] - target_file = os.path.join(target_dir, case_name) - - if skip_exist and os.path.exists(target_file): - log.warning(f"skip case file {target_file}, already exist.") - return - - log.info(f"Write test case to path: {target_dir}, case name is {case_name}") - - content_output = "" - f = open(sql_file, encoding="UTF-8") - sql_content = "" - for line in f.readlines(): - if is_empty_line(line): - continue - - if line.startswith("--"): # pass comment - continue - - # multi line sql - sql_content = sql_content + line.rstrip() - if ";" not in line: - continue - - statement = sql_content.strip() - sql_content = "" - - # error statement - errorStatment = get_error_statment(statement) - if errorStatment is not None: - content_output = content_output + STATEMENT_ERROR.format( - error_id=errorStatment.group("expectError"), - statement=errorStatment.group("statement"), - ) - continue - - if str.lower(first_word(statement)) in query_statment_first_words: - # query statement - - try: - http_results = format_result(http_client.fetch_all(statement)) - query_options = http_client.get_query_option() - except Exception: - exception_sqls.append(statement) - log.error(f"Exception SQL: {statement}") - continue - - if query_options == "" or is_empty_line(http_results): - log.warning( - f"statement: {statement} type query could not get query_option change to ok statement" - ) - content_output = content_output + STATEMENT_OK.format( - statement=statement - ) - continue - - mysql_results = mysql_fetch_results(statement) - query_options_with_labels = f"{query_options}" - - log.debug("sql: " + statement) - log.debug("mysql return: " + mysql_results) - log.debug("http return: " + http_results) - - if http_results is not None and mysql_results != http_results: - case_results = RESULTS_TEMPLATE.format( - results_string=mysql_results, label_separate="----" - ) - - case_results = ( - case_results - + "\n" - + RESULTS_TEMPLATE.format( - results_string=http_results, label_separate="---- http" - ) - ) - - query_options_with_labels = f"{query_options} label(http)" - else: - case_results = RESULTS_TEMPLATE.format( - results_string=mysql_results, label_separate="----" - ) - - content_output = content_output + STATEMENT_QUERY.format( - query_options_with_labels=query_options_with_labels, - statement=statement, - results=case_results, - ) - else: - # ok statement - try: - # insert,drop,create does not need to execute by different handlers - if str.lower(statement).split()[0] not in ["insert", "drop", "create"]: - http_client.query_with_session(statement) - mysql_client.execute(statement) - except Exception as err: - log.warning(f"statement {statement} execute error,msg {str(err)}") - pass - - content_output = content_output + STATEMENT_OK.format(statement=statement) - - f.close() - if not os.path.exists(target_dir): - os.makedirs(target_dir) - - caseFile = open(target_file, "w", encoding="UTF-8") - caseFile.write(content_output) - caseFile.close() - - -def output(): - print("=================================") - print("Exception sql using Http handler:") - print("\n".join(exception_sqls)) - print("=================================") - print("\n") - print("=================================") - print("Manual suites:") - print("\n".join(manual_cases)) - print("=================================") - - -def main(pattern=".*"): - log.debug(f"Case filter regex pattern is {pattern}") - all_cases = get_all_cases() - - for file in all_cases: - if not re.match(pattern, file): - log.debug(f"Test file {file} does not match pattern {pattern}") - continue - log.info(f"Test file {file} match pattern {pattern}") - - # .result will be ignore - if ".result" in file or ".result_filter" in file: - continue - - # .py .sh will be ignore, need log - if ".py" in file or ".sh" in file: - manual_cases.append(file) - log.warning(f"test file {file} will be ignore") - continue - - parse_cases(file) - time.sleep(0.01) - - output() - - -if __name__ == "__main__": - log.info( - f"Start generate sqllogictest suites from path: {suite_path} to path: {logictest_path}" - ) - fire.Fire(main) diff --git a/tests/logictest/http_connector.py b/tests/logictest/http_connector.py deleted file mode 100644 index 9be8b53f9febf..0000000000000 --- a/tests/logictest/http_connector.py +++ /dev/null @@ -1,199 +0,0 @@ -import json -import os -import base64 -import time - -import environs -import requests -from mysql.connector.errors import Error -from log import log - -headers = {"Content-Type": "application/json", "Accept": "application/json"} - -default_database = "default" - - -def format_result(results): - res = "" - if results is None: - return "" - - for line in results: - buf = "" - for item in line: - if isinstance(item, bool): - item = str.lower(str(item)) - if buf == "": - buf = str(item) - else: - buf = buf + " " + str(item) # every item seperate by space - if len(buf) == 0: - # empty line in results will replace with tab - buf = "\t" - res = res + buf + "\n" - return res - - -def get_data_type(field): - if "data_type" in field: - if "inner" in field["data_type"]: - return field["data_type"]["inner"]["type"] - else: - return field["data_type"]["type"] - - -def get_query_options(response): - ret = "" - if get_error(response) is not None: - return ret - for field in response["schema"]["fields"]: - typ = str.lower(get_data_type(field)) - log.debug(f"type:{typ}") - if "int" in typ: - ret = ret + "I" - elif "float" in typ or "double" in typ: - ret = ret + "F" - elif "bool" in typ: - ret = ret + "B" - else: - ret = ret + "T" - return ret - - -def get_next_uri(response): - if "next_uri" in response: - return response["next_uri"] - return None - - -def get_result(response): - return response["data"] - - -def get_error(response): - if response["error"] is None: - return None - - # Wrap errno into msg, for result check - return Error(msg=response["error"]["message"], errno=response["error"]["code"]) - - -def check_error(response): - error = get_error(response) - if error: - raise error - - -class HttpConnector(object): - # Databend http handler doc: https://databend.rs/doc/reference/api/rest - - # Call connect(**driver) - # driver is a dict contains: - # { - # 'user': 'root', - # 'host': '127.0.0.1', - # 'port': 3307, - # 'database': 'default' - # } - def __init__(self, host, port, user="root", database=default_database): - self._host = host - self._port = port - self._user = user - self._database = database - self._session_max_idle_time = 30 - self._session = {} - self._additional_headers = dict() - self._query_option = None - e = environs.Env() - if os.getenv("ADDITIONAL_HEADERS") is not None: - self._additional_headers = e.dict("ADDITIONAL_HEADERS") - - def make_headers(self): - if "Authorization" not in self._additional_headers: - return { - **headers, - "Authorization": "Basic " - + base64.b64encode( - "{}:{}".format(self._user, "").encode(encoding="utf-8") - ).decode(), - } - else: - return {**headers, **self._additional_headers} - - def query(self, statement, session): - url = f"http://{self._host}:{self._port}/v1/query/" - log.debug(f"http sql: {statement}") - - query_sql = {"sql": statement} - - if session is not None: - query_sql["session"] = session - log.debug(f"http headers {self.make_headers()}") - response = requests.post( - url, data=json.dumps(query_sql), headers=self.make_headers() - ) - - try: - return json.loads(response.content) - except Exception as err: - log.error( - f"http error, SQL: {statement}\ncontent: {response.content}\nerror msg:{str(err)}" - ) - raise - - def reset_session(self): - self._session = {} - - def next_page(self, next_uri): - url = "http://{}:{}{}".format(self._host, self._port, next_uri) - return requests.get(url=url, headers=self.make_headers()) - - # return a list of response util empty next_uri - def query_with_session(self, statement): - current_session = self._session - response_list = list() - response = self.query(statement, current_session) - log.debug(f"response content: {response}") - response_list.append(response) - start_time = time.time() - time_limit = 12 - session = response["session"] - if session: - self._session = session - while response["next_uri"] is not None: - resp = self.next_page(response["next_uri"]) - response = json.loads(resp.content) - log.debug(f"Sql in progress, fetch next_uri content: {response}") - check_error(response) - session = response["session"] - if session: - self._session = session - response_list.append(response) - if time.time() - start_time > time_limit: - log.warning( - f"after waited for {time_limit} secs, query still not finished (next uri not none)!" - ) - return response_list - - def fetch_all(self, statement): - resp_list = self.query_with_session(statement) - if len(resp_list) == 0: - log.warning("fetch all with empty results") - return None - self._query_option = get_query_options(resp_list[0]) # record schema - data_list = list() - for response in resp_list: - data = get_result(response) - if len(data) != 0: - data_list.extend(data) - return data_list - - def get_query_option(self): - return self._query_option - - -# if __name__ == '__main__': -# from config import http_config -# connector = HttpConnector() -# connector.connect(**http_config) -# connector.query_without_session("show databases;") diff --git a/tests/logictest/http_runner.py b/tests/logictest/http_runner.py deleted file mode 100644 index 85c20a9d8ea03..0000000000000 --- a/tests/logictest/http_runner.py +++ /dev/null @@ -1,80 +0,0 @@ -from abc import ABC - -import logictest -import http_connector -from mysql.connector.errors import Error - - -class TestHttp(logictest.SuiteRunner, ABC): - def __init__(self, kind, args): - super().__init__(kind, args) - self._http = None - - def get_connection(self): - if self._http is None: - self._http = http_connector.HttpConnector(**self.driver) - return self._http - - def reset_connection(self): - if self._http is not None: - self._http.reset_session() - - def batch_execute(self, statement_list): - for statement in statement_list: - self.execute_statement(statement) - self.reset_connection() - - def execute_ok(self, statement): - try: - self.get_connection().query_with_session(statement) - except Error as e: - return e - - def execute_error(self, statement): - try: - self.get_connection().query_with_session(statement) - except Error as e: - return e - - def execute_query(self, statement): - results = self.get_connection().fetch_all(statement.text) - # query_type = statement.s_type.query_type - vals = [] - for (ri, row) in enumerate(results): - for (i, v) in enumerate(row): - if isinstance(v, type(None)): - vals.append("NULL") - continue - - # todo(youngsofun) : check the schema instead - # if query_type[i] == 'I': - # if not isinstance(v, int): - # log.error( - # "Expected int, got type {} in query {} row {} col {} value {}" - # .format(type(v), statement.text, ri, i, v)) - # elif query_type[i] == 'F' or query_type[i] == 'R': - # if not isinstance(v, float): - # log.error( - # "Expected float, got type {} in query {} row {} col {} value {}" - # .format(type(v), statement.text, ri, i, v)) - # elif query_type[i] == 'T': - # # include data, timestamp, dict, list ... - # if not (isinstance(v, str) or isinstance(v, dict) or - # isinstance(v, list)): - # log.error( - # "Expected string, got type {} in query {} row {} col {} value {}" - # .format(type(v), statement.text, ri, i, v)) - # elif query_type[i] == 'B': - # if not isinstance(v, bool): - # log.error( - # "Expected bool, got type {} in query {} row {} col {} value {}" - # .format(type(v), statement.text, ri, i, v)) - # else: - # log.error( - # "Unknown type {} in query {} row {} col {} value {}". - # format(query_type[i], statement.text, ri, i, v)) - # if isinstance(v, bool): - # v = str(v).lower( - # ) # bool to string in python will be True/False - vals.append(str(v)) - return vals diff --git a/tests/logictest/log.py b/tests/logictest/log.py deleted file mode 100644 index d3aa758738ddd..0000000000000 --- a/tests/logictest/log.py +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: UTF-8 -*- - -import logging - -logging.basicConfig(level=logging.INFO) -log = logging.getLogger(__name__) diff --git a/tests/logictest/logictest.py b/tests/logictest/logictest.py deleted file mode 100644 index 6542494789f89..0000000000000 --- a/tests/logictest/logictest.py +++ /dev/null @@ -1,496 +0,0 @@ -import abc -import collections -import glob -import os -import re -import time -import traceback - -import six - -from log import log -from statistics import global_statistics -from regex_type import compare_result_with_reg - -supports_labels = ["http", "mysql", "clickhouse"] - -# statement is a statement in sql logic test -state_regex = ( - r"^\s*statement\s+(?P((?POK)|((?P)ERROR\s*(?P.*))|(?PQUERY\s*((" - r"ERROR\s+(?P.*))|(?P.*)))))$" -) - -result_regex = r"^----\s*(?P