Skip to content

Commit

Permalink
[Github CI] Enable Oracle database tests
Browse files Browse the repository at this point in the history
Signed-off-by: Christian Parpart <[email protected]>
  • Loading branch information
christianparpart committed Jan 5, 2025
1 parent dd7dc01 commit 9552d24
Show file tree
Hide file tree
Showing 13 changed files with 255 additions and 60 deletions.
21 changes: 13 additions & 8 deletions .github/prepare-test-run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -144,14 +144,19 @@ setup_oracle() {
local DB_USER=$USER
local target_dir="$HOME/oracle"

# Override DB_NAME to FREEPDB1, as this is already created in the Oracle Docker image
# and it helps avoiding unnecessary database creation time.
DB_NAME="FREEPDB1"

set -ex

# References
# - https://github.com/gvenzl/oci-oracle-free

# install Oracle SQL server on ubuntu
docker pull gvenzl/oracle-free:$ORACLE_VERSION
docker run -d -p 1521:1521 \
-e ORACLE_PASSWORD="$DB_PASSWORD" \
-e ORACLE_DATABASE="$DB_NAME" \
-e APP_USER="$DB_USER" \
-e APP_USER_PASSWORD="$DB_PASSWORD" \
gvenzl/oracle-free:$ORACLE_VERSION
Expand All @@ -176,23 +181,23 @@ setup_oracle() {
sudo cp -v etc/odbcinst.ini /etc/

sudo apt install -y libaio-dev
sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/local/lib/libaio.so.1
ldconfig
sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/x86_64-linux-gnu/libaio.so.1
sudo ldconfig

sudo odbcinst -q -d
sudo odbcinst -q -d -n "Oracle ${oracle_odbc_ver_major} ODBC driver"
# }}}

# test connection with:
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${target_dir}/instantclient_21_3
${target_dir}/instantclient_21_3/sqlplus $DB_USER/$DB_PASSWORD@localhost:1521/$DB_NAME <<EOF
SELECT table_name FROM user_tables WHERE ROWNUM <= 5;
EOF
ldd ${target_dir}/instantclient_21_3/sqlplus
DB_USER=system
echo "SELECT table_name FROM user_tables WHERE ROWNUM <= 5;" \
| ${target_dir}/instantclient_21_3/sqlplus -S "$DB_USER/$DB_PASSWORD@localhost:1521/$DB_NAME"

echo "Exporting ODBC_CONNECTION_INFO..."
# expose the ODBC connection string to connect to the database
# echo "ODBC_CONNECTION_STRING=DRIVER=Oracle ${oracle_odbc_ver_major} ODBC driver;SERVER=localhost;PORT=1521;UID=$DB_USER;PWD=$DB_PASSWORD;DBQ=$DB_NAME" >> "${GITHUB_OUTPUT}"
echo "ODBC_CONNECTION_STRING=DRIVER=Oracle ${oracle_odbc_ver_major} ODBC driver;SERVER=localhost;PORT=1521;UID=system;PWD=$DB_PASSWORD;DBA=W" >> "${GITHUB_OUTPUT}"
echo "ODBC_CONNECTION_STRING=DRIVER=Oracle ${oracle_odbc_ver_major} ODBC driver;SERVER=localhost;PORT=1521;UID=$DB_USER;PWD=$DB_PASSWORD;DBQ=$DB_NAME;DBA=W" >> "${GITHUB_OUTPUT}"
}

setup_mysql() {
Expand Down
28 changes: 16 additions & 12 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -245,12 +245,12 @@ jobs:
- name: "Setup ${{ matrix.database }}"
id: setup
run: bash ./.github/prepare-test-run.sh "${{ matrix.database }}"
# Don't do ODBC tracing for now
# - name: "Enable ODBC tracing"
# run: |
# echo "[ODBC]" | sudo tee -a /etc/odbcinst.ini
# echo "Trace=Yes" | sudo tee -a /etc/odbcinst.ini
# echo "TraceFile=/dev/stdout" | sudo tee -a /etc/odbcinst.ini
- name: "Enable ODBC tracing"
if: ${{ matrix.database == 'Oracle' }}
run: |
echo "[ODBC]" | sudo tee -a /etc/odbcinst.ini
echo "Trace=Yes" | sudo tee -a /etc/odbcinst.ini
echo "TraceFile=/dev/stdout" | sudo tee -a /etc/odbcinst.ini
- name: "~/.odbc.ini: set ServerName"
if: ${{ matrix.database == 'Oracle' }}
run: |
Expand All @@ -260,15 +260,19 @@ jobs:
run: |
echo "ODBC_CONNECTION_STRING=${{ steps.setup.outputs.ODBC_CONNECTION_STRING }}"
ldd /home/runner/oracle/instantclient_21_3/libsqora.so.21.1
- name: "Inspect dependencies"
run: ldd /usr/local/bin/LightweightTest
- name: "Run SQL tests"
env:
ODBC_CONNECTION_STRING: "${{ steps.setup.outputs.ODBC_CONNECTION_STRING }}"
run: |
# Don't do --error-exitcode=1 for now
valgrind \
--leak-check=full \
--leak-resolution=high \
--num-callers=64 \
LightweightTest
set -ex
if [[ "${{ matrix.database }}" = "SQLite3" ]]; then
CMD_PREFIX="valgrind --leak-check=full --leak-resolution=high --num-callers=64 --error-exitcode=1"
elif [[ "${{ matrix.database }}" = "Oracle" ]]; then
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:~/oracle/instantclient_21_3
CMD_PREFIX="strace -f"
fi
$CMD_PREFIX LightweightTest
# }}}
2 changes: 1 addition & 1 deletion .vimspector.json
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
"environment": [
{
"name": "ODBC_CONNECTION_STRING",
"value": "DRIVER=Oracle 21 ODBC driver;SERVER=localhost;PORT=1521;UID=system;PWD=Super1Secret.;DBA=W"
"value": "DRIVER=Oracle 21 ODBC driver;SERVER=localhost;PORT=1521;UID=system;PWD=Super1Secret.;DBA=W;Database=FREEPDB1"
}
],
"cwd": "${workspaceRoot}",
Expand Down
7 changes: 7 additions & 0 deletions docs/best-practices.md
Original file line number Diff line number Diff line change
Expand Up @@ -50,3 +50,10 @@ of parsing, analyzing, and compiling the SQL queries.

When querying large result sets, use pagination to limit the number of results returned in a single response.
This will help to reduce the response time and the load on the server.

## SQL server variation challenges

### 64-bit integer handling in Oracle database

Oracle database does not support 64-bit integers natively.
When working with 64-bit integers in Oracle database, you need to use the `SqlNumeric` column types.
1 change: 1 addition & 0 deletions src/Lightweight/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ set(HEADER_FILES
)

set(SOURCE_FILES
DataBinder/Primitives.cpp
DataBinder/SqlGuid.cpp
DataBinder/SqlVariant.cpp
DataBinder/UnicodeConverter.cpp
Expand Down
123 changes: 123 additions & 0 deletions src/Lightweight/DataBinder/Primitives.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
// SPDX-License-Identifier: Apache-2.0

#include "BasicStringBinder.hpp"
#include "Primitives.hpp"
#include "SqlFixedString.hpp"

#include <array>

template <typename Int64Type, SQLSMALLINT TheCType>
struct OracleInt64DataBinderHelper
{
static SQLRETURN InputParameter(SQLHSTMT stmt,
SQLUSMALLINT column,
Int64Type const& value,
SqlDataBinderCallback& cb) noexcept
{
using StringType = SqlFixedString<21>;
auto strValue = std::make_shared<StringType>();
std::to_chars(strValue->data(), strValue->data() + strValue->capacity(), value, 10);
cb.PlanPostExecuteCallback([strValue]() {}); // Defer the destruction of the string until after the execute
return SqlDataBinder<StringType>::InputParameter(stmt, column, *strValue, cb);
}

static SQLRETURN OutputColumn(
SQLHSTMT stmt, SQLUSMALLINT column, Int64Type* result, SQLLEN* indicator, SqlDataBinderCallback& cb) noexcept
{
using StringType = SqlFixedString<21>;
auto buffer = std::make_shared<StringType>();
auto const sqlResult = SqlDataBinder<StringType>::OutputColumn(stmt, column, buffer.get(), indicator, cb);
if (SQL_SUCCEEDED(sqlResult))
{
cb.PlanPostProcessOutputColumn([buffer, result, indicator]() {
if (*indicator != SQL_NULL_DATA && *indicator != SQL_NO_TOTAL)
{
std::from_chars(buffer->data(), buffer->data() + buffer->size(), *result, 10);
}
});
}
return sqlResult;
}

static SQLRETURN GetColumn(SQLHSTMT stmt,
SQLUSMALLINT column,
Int64Type* result,
SQLLEN* indicator,
SqlDataBinderCallback const& cb) noexcept
{
using StringType = SqlFixedString<21>;
auto buffer = StringType {};
auto const sqlResult = SqlDataBinder<StringType>::GetColumn(stmt, column, &buffer, indicator, cb);
if (SQL_SUCCEEDED(sqlResult) && *indicator != SQL_NULL_DATA && *indicator != SQL_NO_TOTAL)
{
std::from_chars(buffer.data(), buffer.data() + buffer.size(), *result, 10);
}
return sqlResult;
}
};

// We use jump tables below to avoid CPU branch prediction misses.

template <typename Int64Type, SQLSMALLINT TheCType>
SQLRETURN Int64DataBinderHelper<Int64Type, TheCType>::InputParameter(SQLHSTMT stmt,
SQLUSMALLINT column,
Int64Type const& value,
SqlDataBinderCallback& cb) noexcept
{
// clang-format off
static constexpr auto map = std::array {
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::InputParameter, // Unknown
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::InputParameter, // Microsoft SQL
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::InputParameter, // PostgreSQL
&OracleInt64DataBinderHelper<Int64Type, TheCType>::InputParameter,
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::InputParameter, // SQLite
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::InputParameter, // MySQL
};
// clang-format on
auto const index = static_cast<size_t>(cb.ServerType());
return map[index](stmt, column, value, cb);
}

template <typename Int64Type, SQLSMALLINT TheCType>
SQLRETURN Int64DataBinderHelper<Int64Type, TheCType>::OutputColumn(
SQLHSTMT stmt, SQLUSMALLINT column, Int64Type* result, SQLLEN* indicator, SqlDataBinderCallback& cb) noexcept
{
// clang-format off
static constexpr auto map = std::array {
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::OutputColumn, // Unknown
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::OutputColumn, // Microsoft SQL
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::OutputColumn, // PostgreSQL
&OracleInt64DataBinderHelper<Int64Type, TheCType>::OutputColumn,
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::OutputColumn, // SQLite
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::OutputColumn, // MySQL
};
// clang-format on
auto const index = static_cast<size_t>(cb.ServerType());
return map[index](stmt, column, result, indicator, cb);
}

template <typename Int64Type, SQLSMALLINT TheCType>
SQLRETURN Int64DataBinderHelper<Int64Type, TheCType>::GetColumn(
SQLHSTMT stmt, SQLUSMALLINT column, Int64Type* result, SQLLEN* indicator, SqlDataBinderCallback const& cb) noexcept
{
// clang-format off
static constexpr auto map = std::array {
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::GetColumn, // Unknown
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::GetColumn, // Microsoft SQL
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::GetColumn, // PostgreSQL
&OracleInt64DataBinderHelper<Int64Type, TheCType>::GetColumn,
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::GetColumn, // SQLite
&SqlSimpleDataBinder<Int64Type, TheCType, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}>::GetColumn, // MySQL
};
// clang-format on
auto const index = static_cast<size_t>(cb.ServerType());
return map[index](stmt, column, result, indicator, cb);
}

template struct Int64DataBinderHelper<int64_t, SQL_C_SBIGINT>;
template struct Int64DataBinderHelper<uint64_t, SQL_C_UBIGINT>;

#if !defined(_WIN32) && !defined(__APPLE__)
template struct Int64DataBinderHelper<long long, SQL_C_SBIGINT>;
template struct Int64DataBinderHelper<unsigned long long, SQL_C_UBIGINT>;
#endif
34 changes: 30 additions & 4 deletions src/Lightweight/DataBinder/Primitives.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -37,20 +37,46 @@ struct SqlSimpleDataBinder
}
};

template <typename Int64Type, SQLSMALLINT TheCType>
struct Int64DataBinderHelper
{
static constexpr SqlColumnTypeDefinition ColumnType = SqlColumnTypeDefinitions::Bigint {};

static LIGHTWEIGHT_API SQLRETURN InputParameter(SQLHSTMT stmt,
SQLUSMALLINT column,
Int64Type const& value,
SqlDataBinderCallback& cb) noexcept;

static LIGHTWEIGHT_API SQLRETURN OutputColumn(
SQLHSTMT stmt, SQLUSMALLINT column, Int64Type* result, SQLLEN* indicator, SqlDataBinderCallback& cb) noexcept;

static LIGHTWEIGHT_API SQLRETURN GetColumn(SQLHSTMT stmt,
SQLUSMALLINT column,
Int64Type* result,
SQLLEN* indicator,
SqlDataBinderCallback const& cb) noexcept;

static LIGHTWEIGHT_FORCE_INLINE std::string Inspect(Int64Type value)
{
return std::to_string(value);
}
};

// clang-format off
template <> struct SqlDataBinder<bool>: SqlSimpleDataBinder<bool, SQL_BIT, SQL_BIT, SqlColumnTypeDefinitions::Bool {}> {};
template <> struct SqlDataBinder<char>: SqlSimpleDataBinder<char, SQL_C_CHAR, SQL_CHAR, SqlColumnTypeDefinitions::Char {}> {};
template <> struct SqlDataBinder<int16_t>: SqlSimpleDataBinder<int16_t, SQL_C_SSHORT, SQL_SMALLINT, SqlColumnTypeDefinitions::Smallint {}> {};
template <> struct SqlDataBinder<uint16_t>: SqlSimpleDataBinder<uint16_t, SQL_C_USHORT, SQL_SMALLINT, SqlColumnTypeDefinitions::Smallint {}> {};
template <> struct SqlDataBinder<int32_t>: SqlSimpleDataBinder<int32_t, SQL_C_SLONG, SQL_INTEGER, SqlColumnTypeDefinitions::Integer {}> {};
template <> struct SqlDataBinder<uint32_t>: SqlSimpleDataBinder<uint32_t, SQL_C_ULONG, SQL_INTEGER, SqlColumnTypeDefinitions::Integer {}> {};
template <> struct SqlDataBinder<int64_t>: SqlSimpleDataBinder<int64_t, SQL_C_SBIGINT, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}> {};
template <> struct SqlDataBinder<uint64_t>: SqlSimpleDataBinder<uint64_t, SQL_C_UBIGINT, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}> {};
template <> struct SqlDataBinder<int64_t>: Int64DataBinderHelper<int64_t, SQL_C_SBIGINT> {};
template <> struct SqlDataBinder<uint64_t>: Int64DataBinderHelper<uint64_t, SQL_C_UBIGINT> {};
//template <> struct SqlDataBinder<uint64_t>: Int64DataBinderHelper<uint64_t, SQL_C_UBIGINT> {};
template <> struct SqlDataBinder<float>: SqlSimpleDataBinder<float, SQL_C_FLOAT, SQL_REAL, SqlColumnTypeDefinitions::Real {}> {};
template <> struct SqlDataBinder<double>: SqlSimpleDataBinder<double, SQL_C_DOUBLE, SQL_DOUBLE, SqlColumnTypeDefinitions::Real {}> {};
#if !defined(_WIN32) && !defined(__APPLE__)
template <> struct SqlDataBinder<long long>: SqlSimpleDataBinder<long long, SQL_C_SBIGINT, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}> {};
template <> struct SqlDataBinder<unsigned long long>: SqlSimpleDataBinder<unsigned long long, SQL_C_UBIGINT, SQL_BIGINT, SqlColumnTypeDefinitions::Bigint {}> {};
template <> struct SqlDataBinder<long long>: Int64DataBinderHelper<long long, SQL_C_SBIGINT> {};
template <> struct SqlDataBinder<unsigned long long>: Int64DataBinderHelper<unsigned long long, SQL_C_UBIGINT> {};
#endif
#if defined(__APPLE__) // size_t is a different type on macOS
template <> struct SqlDataBinder<std::size_t>: SqlSimpleDataBinder<std::size_t, SQL_C_SBIGINT, SqlColumnTypeDefinitions::Bigint {}> {};
Expand Down
9 changes: 2 additions & 7 deletions src/Lightweight/QueryFormatter/OracleFormatter.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -79,13 +79,8 @@ class OracleSqlQueryFormatter final: public SQLiteQueryFormatter
using namespace SqlColumnTypeDefinitions;
return std::visit(
detail::overloaded {
[](Bigint const&) -> std::string { return "NUMBER(19, 0)"; },
[](Binary const& type) -> std::string {
if (type.size)
return std::format("BLOB({})", type.size);
else
return "BLOB";
},
[](Bigint const&) -> std::string { return "NUMBER(21, 0)"; },
[](Binary const&) -> std::string { return "BLOB"; },
[](Bool const&) -> std::string { return "BIT"; },
[](Char const& type) -> std::string { return std::format("CHAR({})", type.size); },
[](Date const&) -> std::string { return "DATE"; },
Expand Down
2 changes: 2 additions & 0 deletions src/tests/CoreTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -325,6 +325,8 @@ TEST_CASE_METHOD(SqlTestFixture, "LastInsertId", "[SqlStatement]")
{
auto stmt = SqlStatement {};

UNSUPPORTED_DATABASE(stmt, SqlServerType::ORACLE);

CreateEmployeesTable(stmt);
FillEmployeesTable(stmt);

Expand Down
20 changes: 13 additions & 7 deletions src/tests/DataBinderTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -443,13 +443,17 @@ struct TestTypeTraits<int32_t>
template <>
struct TestTypeTraits<int64_t>
{
static constexpr auto blacklist = std::array {
std::pair { SqlServerType::ORACLE, "Oracle does not support 64-bit integers via SQL_BIGINT it seems"sv },
};
static constexpr auto inputValue = (std::numeric_limits<int64_t>::max)();
static constexpr auto expectedOutputValue = (std::numeric_limits<int64_t>::max)();
};

template <>
struct TestTypeTraits<uint64_t>
{
static constexpr auto inputValue = (std::numeric_limits<uint64_t>::max)();
static constexpr auto expectedOutputValue = (std::numeric_limits<uint64_t>::max)();
};

template <>
struct TestTypeTraits<float>
{
Expand Down Expand Up @@ -654,6 +658,9 @@ struct TestTypeTraits<std::wstring_view>
template <>
struct TestTypeTraits<SqlBinary>
{
static constexpr auto blacklist = std::array {
std::pair { SqlServerType::ORACLE, "TODO: Oracle"sv },
};
static constexpr auto sqlColumnTypeNameOverride = SqlColumnTypeDefinitions::Binary { 50 };
static auto const inline inputValue = SqlBinary { 0x00, 0x02, 0x03, 0x00, 0x05 };
static auto const inline expectedOutputValue = SqlBinary { 0x00, 0x02, 0x03, 0x00, 0x05 };
Expand All @@ -678,6 +685,7 @@ using TypesToTest = std::tuple<
int16_t,
int32_t,
int64_t,
// TODO: uint64_t,
std::string,
std::string_view,
std::u16string,
Expand Down Expand Up @@ -763,8 +771,7 @@ TEMPLATE_LIST_TEST_CASE("SqlDataBinder specializations", "[SqlDataBinder]", Type
// {
// stmt.ExecuteDirect("SELECT Value FROM Test");
// auto actualValue = [&]() -> TestType {
// if constexpr (requires(SqlServerType st) { TestTypeTraits<TestType>::outputInitializer(st);
// })
// if constexpr (requires(SqlServerType st) { TestTypeTraits<TestType>::outputInitializer(st); })
// return TestTypeTraits<TestType>::outputInitializer(conn.ServerType());
// else if constexpr (requires { TestTypeTraits<TestType>::outputInitializer; })
// return TestTypeTraits<TestType>::outputInitializer;
Expand All @@ -776,8 +783,7 @@ TEMPLATE_LIST_TEST_CASE("SqlDataBinder specializations", "[SqlDataBinder]", Type
// if constexpr (std::is_convertible_v<TestType, double> && !std::integral<TestType>)
// CHECK_THAT(
// double(actualValue),
// (Catch::Matchers::WithinAbs(double(TestTypeTraits<TestType>::expectedOutputValue),
// 0.001)));
// (Catch::Matchers::WithinAbs(double(TestTypeTraits<TestType>::expectedOutputValue), 0.001)));
// else
// CHECK(actualValue == TestTypeTraits<TestType>::expectedOutputValue);
// }
Expand Down
6 changes: 3 additions & 3 deletions src/tests/DataMapperTests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -728,10 +728,10 @@ TEST_CASE_METHOD(SqlTestFixture, "Table with aliased column names", "[DataMapper

struct PersonDifferenceView
{
Field<SqlGuid, PrimaryKey::AutoAssign> id;
Field<SqlAnsiString<25>> name;
Field<SqlGuid, PrimaryKey::AutoAssign> id {};
Field<SqlAnsiString<25>> name {};
Field<bool> is_active { true };
Field<int> age;
Field<int> age {};
};

TEST_CASE_METHOD(SqlTestFixture, "Test DifferenceView", "[DataMapper]")
Expand Down
Loading

0 comments on commit 9552d24

Please sign in to comment.