GH-47921: [C++] Implement substrait option in Meson

cpp/meson.build

@@ -53,13 +53,14 @@ endif
 
 needs_benchmarks = get_option('benchmarks').enabled()
 needs_csv = get_option('csv').enabled()
-needs_dataset = get_option('dataset').enabled()
+needs_substrait = get_option('substrait').enabled()
+needs_dataset = get_option('dataset').enabled() or needs_substrait
 needs_azure = get_option('azure').enabled()
 needs_gcs = get_option('gcs').enabled()
 needs_hdfs = get_option('hdfs').enabled()
 needs_opentelemetry = false
 needs_orc = false
-needs_parquet = get_option('parquet').enabled()
+needs_parquet = get_option('parquet').enabled() or needs_substrait
 needs_parquet_encryption = get_option('parquet_require_encryption').enabled()
 needs_s3 = get_option('s3').enabled()
 needs_filesystem = (get_option('filesystem').enabled()
@@ -82,6 +83,7 @@ needs_ipc = (get_option('ipc').enabled()
     or needs_benchmarks
     or needs_flight
     or needs_parquet
+    or needs_substrait
 )
 
 needs_fuzzing = get_option('fuzzing').enabled()
@@ -109,6 +111,29 @@ needs_zlib = get_option('zlib').enabled()
 needs_zstd = get_option('zstd').enabled()
 needs_utilities = get_option('utilities').enabled()
 
+if needs_flight or needs_substrait
+    protobuf_dep = dependency('protobuf')
+    protoc = find_program('protoc')
+
+    # To ensure messages from proto files are created correctly, we need to
+    # pass in dllexport_decl=<...> . Unfortunately, it doesn't appear that we
+    # can just pass in dllexport_decl=XXX_EXPORT, as the visibility
+    # macro won't be easily available to the generated proto file. See also
+    # https://github.com/protocolbuffers/protobuf/issues/19422
+    if meson.get_compiler('cpp').get_id() == 'msvc'
+        if get_option('default_library') != 'static'
+            proto_visibility = 'dllexport_decl=__declspec(dllexport):'
+        else
+            proto_visibility = ''
+        endif
+    else
+        proto_visibility = 'dllexport_decl=__attribute__((visibility("default"))):'
+    endif
+else
+    protobuf_dep = disabler()
+    protoc = disabler()
+endif
+
 subdir('src/arrow')
 
 if needs_parquet
@@ -126,3 +151,7 @@ if needs_dataset
     # with a circular dependency
     subdir('src/arrow/dataset')
 endif
+
+if needs_substrait
+    subdir('src/arrow/engine')
+endif

cpp/meson.options

@@ -112,6 +112,11 @@ option(
 )
 
 option('snappy', type: 'feature', description: 'Build with snappy compression')
+option(
+    'substrait',
+    type: 'feature',
+    description: 'Build the Arrow Substrait Consumer Module',
+)
 option(
     's3',
     type: 'feature',

cpp/src/arrow/engine/meson.build

@@ -0,0 +1,109 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+install_headers(['api.h'], subdir: 'arrow/engine')
+
+arrow_substrait_protos_dir = meson.project_source_root() / 'proto'
+substrait_proto_gen = generator(
+    protoc,
+    output: ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'],
+    arguments: [
+        '--proto_path=@0@'.format(
+            subproject('substrait').get_variable('substrait_protos_dir'),
+        ),
+        '--proto_path=@0@'.format(arrow_substrait_protos_dir),
+        '--cpp_out=@0@@1@'.format(proto_visibility, '@BUILD_DIR@'),
+        '@INPUT@',
+    ],
+)
+ext_proto_gen = substrait_proto_gen.process(
+    meson.project_source_root() / 'proto' / 'substrait' / 'extension_rels.proto',
+    preserve_path_from: meson.project_source_root() / 'proto',
+)
+substrait_ext_dep = declare_dependency(sources: ext_proto_gen)
+
+substrait_dep = dependency('substrait')
+arrow_substrait_srcs = files(
+    'substrait/expression_internal.cc',
+    'substrait/extended_expression_internal.cc',
+    'substrait/extension_set.cc',
+    'substrait/extension_types.cc',
+    'substrait/options.cc',
+    'substrait/plan_internal.cc',
+    'substrait/relation_internal.cc',
+    'substrait/serde.cc',
+    'substrait/test_plan_builder.cc',
+    'substrait/type_internal.cc',
+    'substrait/util.cc',
+    'substrait/util_internal.cc',
+)
+arrow_substrait_deps = [
+    substrait_dep,
+    substrait_ext_dep,
+    arrow_dataset_dep,
+    protobuf_dep,
+]
+arrow_substrait_pkgconf_req = ['arrow-dataset']
+
+arrow_substrait_lib = library(
+    'arrow_substrait',
+    sources: arrow_substrait_srcs,
+    dependencies: arrow_substrait_deps,
+    cpp_static_args: ['-DARROW_ENGINE_STATIC'],
+    cpp_shared_args: ['-DARROW_ENGINE_EXPORTING'],
+    gnu_symbol_visibility: 'inlineshidden',
+)
+
+arrow_substrait_args = []
+if get_option('default_library') == 'static'
+    arrow_substrait_args += ['-DARROW_ENGINE_STATIC']
+endif
+arrow_substrait_dep = declare_dependency(
+    link_with: arrow_substrait_lib,
+    dependencies: arrow_substrait_deps,
+    compile_args: arrow_substrait_args,
+)
+meson.override_dependency('arrow-substrait', arrow_substrait_dep)
+
+pkg_config_cflags = []
+if get_option('default_library') == 'static'
+    pkg_config_cflags += ['-DARROW_ENGINE_STATIC']
+endif
+pkg.generate(
+    arrow_substrait_lib,
+    filebase: 'arrow-substrait',
+    name: 'Apache Arrow Substrait Consumer',
+    description: 'Apache Arrow\'s Substrait Consumer.',
+    extra_cflags: [pkg_config_cflags],
+    requires: arrow_substrait_pkgconf_req,
+    variables: {'Cflags.private': '-DARROW_ENGINE_STATIC'},
+)
+
+arrow_substrait_test = executable(
+    'substrait-test',
+    sources: files(
+        'substrait/ext_test.cc',
+        'substrait/function_test.cc',
+        'substrait/protobuf_test_util.cc',
+        'substrait/serde_test.cc',
+        'substrait/test_util.cc',
+    ),
+    dependencies: [arrow_substrait_dep, arrow_compute_test_dep],
+)
+test('substrait-test', arrow_substrait_test)
+
+subdir('substrait')

cpp/src/arrow/engine/substrait/meson.build

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+install_headers(
+    [
+        'api.h',
+        'extension_set.h',
+        'extension_types.h',
+        'options.h',
+        'relation.h',
+        'serde.h',
+        'test_plan_builder.h',
+        'test_util.h',
+        'type_fwd.h',
+        'util.h',
+        'visibility.h',
+    ],
+    subdir: 'arrow/engine/substrait',
+)
+

cpp/src/arrow/flight/meson.build

@@ -45,29 +45,11 @@ install_headers(
 )
 
 grpc_dep = dependency('grpc++')
-protobuf_dep = dependency('protobuf')
 abseil_sync_dep = dependency('absl_synchronization')
 
 fs = import('fs')
-protoc = find_program('protoc')
 
 flight_proto_path = fs.parent(meson.project_source_root()) / 'format'
-
-# To ensure messages from proto files are created correctly, we need to
-# pass in dllexport_decl=<...> . Unfortunately, it doesn't appear that we
-# can just pass in dllexport_decl=ARROW_FLIGHT_EXPORT, as the visibility
-# macro won't be easily available to the generated proto file. See also
-# https://github.com/protocolbuffers/protobuf/issues/19422
-if cpp_compiler.get_id() == 'msvc'
-    if get_option('default_library') != 'static'
-        proto_visibility = 'dllexport_decl=__declspec(dllexport):'
-    else
-        proto_visibility = ''
-    endif
-else
-    proto_visibility = 'dllexport_decl=__attribute__((visibility("default"))):'
-endif
-
 flight_proto_files = custom_target(
     'arrow-flight-proto-files',
     input: [flight_proto_path / 'Flight.proto'],

cpp/src/arrow/util/meson.build

@@ -51,7 +51,7 @@ conf_data.set('ARROW_JSON', needs_json)
 conf_data.set('ARROW_MIMALLOC', false)
 conf_data.set('ARROW_ORC', false)
 conf_data.set('ARROW_PARQUET', needs_parquet)
-conf_data.set('ARROW_SUBSTRAIT', false)
+conf_data.set('ARROW_SUBSTRAIT', needs_substrait)
 conf_data.set('ARROW_AZURE', false)
 conf_data.set('ARROW_ENABLE_THREADING', true)
 conf_data.set('ARROW_GCS', false)

cpp/subprojects/packagefiles/substrait/meson.build

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+project('substrait', 'cpp')
+
+protobuf_dep = dependency('protobuf')
+protoc = find_program('protoc')
+
+# To ensure messages from proto files are created correctly, we need to
+# pass in dllexport_decl=<...> . Unfortunately, it doesn't appear that we
+# can just pass in dllexport_decl=SUBSTRAIT_EXPORT, as the visibility
+# macro won't be easily available to the generated proto file. See also
+# https://github.com/protocolbuffers/protobuf/issues/19422
+if meson.get_compiler('cpp').get_id() == 'msvc'
+    if get_option('default_library') != 'static'
+        proto_visibility = 'dllexport_decl=__declspec(dllexport):'
+    else
+        proto_visibility = ''
+    endif
+else
+    proto_visibility = 'dllexport_decl=__attribute__((visibility("default"))):'
+endif
+
+substrait_protos_dir = meson.current_source_dir() / 'proto'
+gen = generator(
+    protoc,
+    output: ['@BASENAME@.pb.cc', '@BASENAME@.pb.h'],
+    arguments: [
+        '--proto_path=@0@'.format(substrait_protos_dir),
+        '--cpp_out=@0@@1@'.format(proto_visibility, '@BUILD_DIR@'),
+        '@INPUT@',
+    ],
+)
+protos = [
+    'substrait/algebra.proto',
+    'substrait/extended_expression.proto',
+    'substrait/extensions/extensions.proto',
+    'substrait/plan.proto',
+    'substrait/type.proto',
+]
+generated_sources = []
+foreach proto : protos
+    generated_sources += [
+        gen.process(
+            meson.current_source_dir() / 'proto' / proto,
+            preserve_path_from: substrait_protos_dir,
+        ),
+    ]
+endforeach
+
+substrait_dep = declare_dependency(
+    sources: generated_sources,
+    include_directories: include_directories('.'),
+)
+meson.override_dependency('substrait', substrait_dep)

cpp/subprojects/substrait.wrap

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+source_url = https://github.com/substrait-io/substrait/archive/v0.44.0.tar.gz
+source_filename = substrait-0.44.0.tar.gz
+source_hash = f989a862f694e7dbb695925ddb7c4ce06aa6c51aca945105c075139aed7e55a2
+directory = substrait-0.44.0
+patch_directory=substrait
+
+[provide]
+dependency_names = substrait