diff --git a/cpp/src/parquet/meson.build b/cpp/src/parquet/meson.build index e6ff43a0baee..ea741d6727ff 100644 --- a/cpp/src/parquet/meson.build +++ b/cpp/src/parquet/meson.build @@ -89,6 +89,10 @@ endif parquet_deps = [arrow_dep, rapidjson_dep, thrift_dep] +# Default to no DBPA test-agent library (only built when encryption+testing are enabled). +dbpa_test_agent_lib = disabler() +dbpa_test_agent_path = '' + if needs_parquet_encryption or get_option('parquet_require_encryption').auto() openssl_dep = dependency('openssl', required: needs_parquet_encryption) else @@ -99,8 +103,15 @@ if openssl_dep.found() parquet_deps += openssl_dep parquet_srcs += files( + 'encryption/aes_encryption.cc', 'encryption/crypto_factory.cc', - 'encryption/encryption_internal.cc', + 'encryption/encoding_properties.cc', + 'encryption/encryption_utils.cc', + 'encryption/external/dbpa_enum_utils.cc', + 'encryption/external/dbpa_executor.cc', + 'encryption/external/dbpa_library_wrapper.cc', + 'encryption/external/loadable_encryptor_utils.cc', + 'encryption/external_dbpa_encryption.cc', 'encryption/file_key_unwrapper.cc', 'encryption/file_key_wrapper.cc', 'encryption/file_system_key_material_store.cc', @@ -112,8 +123,51 @@ if openssl_dep.found() 'encryption/local_wrap_kms_client.cc', 'encryption/openssl_internal.cc', ) + + # External DBPA integration and its header-only deps are only relevant when + # encryption support is enabled (i.e., OpenSSL is available). + if needs_parquet_encryption or get_option('parquet_require_encryption').auto() + tcb_span_dep = dependency( + 'tcb_span', + fallback: ['tcb-span', 'tcb_span_dep'], + ) + magic_enum_dep = dependency( + 'magic_enum_header_only', + fallback: ['magic-enum', 'magic_enum_dep'], + ) + + # DBPS interface is header-only (dbpa_interface.h and friends). + # + # We do not build DBPS agents as part of Arrow's Meson build. We only need the + # DBPS interface headers (dbpa_interface.h, enums.h, etc.) for compilation. + # These are obtained via Meson's wrap subproject `dbps_agent`, which downloads + # and unpacks DBPS sources, and exposes only the interface headers via a tiny + # header-only Meson wrapper (cpp/subprojects/packagefiles/dbps_agent). + dbps_sp = subproject('dbps_agent') + dbps_interface_dep = dbps_sp.get_variable('dbps_interface_dep') + + parquet_deps += [dbps_interface_dep, tcb_span_dep, magic_enum_dep] + + # Build the in-tree DBPA test agent shared library used by external encryption tests. + # CMake builds this as `DBPATestAgent` when ARROW_TESTING is enabled; Meson needs an + # equivalent target so tests can dlopen() `libDBPATestAgent.so`. + if needs_testing + dbpa_test_agent_lib = shared_library( + 'DBPATestAgent', + sources: files('encryption/external/dbpa_test_agent.cc'), + include_directories: include_directories('..'), + # Keep this test agent as self-contained as possible. It is dlopen()'d + # by tests, so avoid unnecessary runtime dependencies (e.g., libarrow.so) + # which can differ between Meson/CMake CI environments. + dependencies: [magic_enum_dep, tcb_span_dep, dbps_interface_dep], + install: false, + gnu_symbol_visibility: 'default', + ) + dbpa_test_agent_path = dbpa_test_agent_lib.full_path() + endif + endif else - parquet_srcs += files('encryption/encryption_internal_nossl.cc') + parquet_srcs += files('encryption/aes_encryption_nossl.cc') endif parquet_lib = library( @@ -123,7 +177,10 @@ parquet_lib = library( gnu_symbol_visibility: 'inlineshidden', ) -parquet_dep = declare_dependency(link_with: parquet_lib) +parquet_dep = declare_dependency( + link_with: parquet_lib, + dependencies: parquet_deps, +) subdir('api') subdir('arrow') @@ -216,6 +273,7 @@ parquet_tests = { 'writer-test': { 'sources': files( 'column_writer_test.cc', + 'encryption/external/test_utils.cc', 'file_serialize_test.cc', 'stream_writer_test.cc', ), @@ -226,6 +284,7 @@ parquet_tests = { 'arrow/arrow_reader_writer_test.cc', 'arrow/arrow_statistics_test.cc', 'arrow/variant_test.cc', + 'encryption/external/test_utils.cc', ), }, 'arrow-internals-test': { @@ -240,7 +299,12 @@ parquet_tests = { 'arrow/arrow_schema_test.cc', ), }, - 'file_deserialize_test': {'sources': files('file_deserialize_test.cc')}, + 'file_deserialize_test': { + 'sources': files( + 'encryption/external/test_utils.cc', + 'file_deserialize_test.cc', + ), + }, 'schema_test': {'sources': files('schema_test.cc')}, } @@ -248,10 +312,16 @@ if needs_parquet_encryption parquet_tests += { 'encryption-test': { 'sources': files( - 'encryption/encryption_internal_test.cc', + 'encryption/aes_encryption_test.cc', + 'encryption/crypto_factory_test.cc', + 'encryption/encoding_properties_test.cc', + 'encryption/external/test_utils.cc', + 'encryption/external_dbpa_encryption_test.cc', + 'encryption/per_column_encryption_test.cc', 'encryption/properties_test.cc', 'encryption/read_configurations_test.cc', 'encryption/test_encryption_util.cc', + 'encryption/test_in_memory_kms.cc', 'encryption/write_configurations_test.cc', ), }, @@ -297,7 +367,17 @@ foreach key, val : parquet_tests sources: val['sources'] + files('test_util.cc'), dependencies: parquet_test_dep, ) - test(test_name, exc) + # Ensure the DBPA test agent is built before running any tests that may load it. + # (No-op when dbpa_test_agent_lib is disabled/unset.) + test_env = { + # Make DBPATestAgent lookup deterministic under Meson. Some CI setups may + # not allow /proc/self/exe resolution or run tests with unexpected cwd. + 'PARQUET_TEST_LIBRARY_CWD': meson.current_build_dir(), + 'DBPA_LIBRARY_PATH': dbpa_test_agent_path, + # Hardcode DBPA logging for Meson test runs to aid CI diagnostics. + 'PARQUET_DBPA_LOG_LEVEL': 'INFO', + } + test(test_name, exc, depends: dbpa_test_agent_lib, env: test_env) endforeach parquet_benchmarks = { diff --git a/cpp/subprojects/dbps_agent.wrap b/cpp/subprojects/dbps_agent.wrap new file mode 100644 index 000000000000..1cdd4d31d2c5 --- /dev/null +++ b/cpp/subprojects/dbps_agent.wrap @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = DataBatchProtectionService-6206fb0e27556a0df9160364caa3819e4af3fe0f +source_url = https://github.com/protegrity/DataBatchProtectionService/archive/6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz +source_filename = dbps_agent-6206fb0e27556a0df9160364caa3819e4af3fe0f.tar.gz +source_hash = 9c95a1fec0c9851867a776c3241d3feb59b07bd7a50e653d6214e07a8ad62419 +patch_directory = dbps_agent + diff --git a/cpp/subprojects/magic-enum.wrap b/cpp/subprojects/magic-enum.wrap new file mode 100644 index 000000000000..5e4b7c7da105 --- /dev/null +++ b/cpp/subprojects/magic-enum.wrap @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = magic_enum-0.9.7 +source_url = https://github.com/Neargye/magic_enum/archive/refs/tags/v0.9.7.tar.gz +source_filename = magic_enum-0.9.7.tar.gz +source_hash = b403d3dad4ef542fdc3024fa37d3a6cedb4ad33c72e31b6d9bab89dcaf69edf7 +patch_directory = magic-enum + +[provide] +magic_enum_header_only = magic_enum_dep diff --git a/cpp/subprojects/packagefiles/dbps_agent/meson.build b/cpp/subprojects/packagefiles/dbps_agent/meson.build new file mode 100644 index 000000000000..d30433ba6036 --- /dev/null +++ b/cpp/subprojects/packagefiles/dbps_agent/meson.build @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('dbps_agent', 'cpp') + +# Expose the DBPS interface headers (dbpa_interface.h, enums.h, etc.). +# This subproject is intentionally header-only: Arrow's Meson build must not +# build DBPS agents or shared libraries. +dbps_interface_dep = declare_dependency( + include_directories: include_directories('src/common'), +) + +# Optional: allow `dependency('dbps_interface')` elsewhere if needed. +meson.override_dependency('dbps_interface', dbps_interface_dep) + diff --git a/cpp/subprojects/packagefiles/magic-enum/meson.build b/cpp/subprojects/packagefiles/magic-enum/meson.build new file mode 100644 index 000000000000..fd877d94704f --- /dev/null +++ b/cpp/subprojects/packagefiles/magic-enum/meson.build @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('magic-enum', 'cpp') + +magic_enum_dep = declare_dependency( + include_directories: include_directories('include'), +) + +meson.override_dependency('magic_enum_header_only', magic_enum_dep) diff --git a/cpp/subprojects/packagefiles/tcb-span/meson.build b/cpp/subprojects/packagefiles/tcb-span/meson.build new file mode 100644 index 000000000000..99966eba78e8 --- /dev/null +++ b/cpp/subprojects/packagefiles/tcb-span/meson.build @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +project('tcb-span', 'cpp') + +tcb_span_dep = declare_dependency( + include_directories: include_directories('include'), +) + +meson.override_dependency('tcb_span', tcb_span_dep) diff --git a/cpp/subprojects/tcb-span.wrap b/cpp/subprojects/tcb-span.wrap new file mode 100644 index 000000000000..fbe7e9a0e523 --- /dev/null +++ b/cpp/subprojects/tcb-span.wrap @@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[wrap-file] +directory = span-master +source_url = https://github.com/tcbrindle/span/archive/refs/heads/master.tar.gz +source_filename = span-master.tar.gz +source_hash = e57658bd9c5984cebd8bd0758f242cbf4fa7ed1bd5251a883f68420c3b17f818 +patch_directory = tcb-span + +[provide] +tcb_span = tcb_span_dep