feat(CB_GF): add reduction (VowpalWabbit#4560)

.github/workflows/build_macos.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Install dependencies
         run: brew install cmake boost flatbuffers ninja
       - name: Configure
-        run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DWARNINGS=Off -DVW_BUILD_VW_C_WRAPPER=Off -DBUILD_TESTING=On -DBUILD_EXPERIMENTAL_BINDING=On -DVW_FEAT_CSV=On -DVW_INSTALL=Off
+        run: cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=${{matrix.build_type}} -DWARNINGS=Off -DVW_BUILD_VW_C_WRAPPER=Off -DBUILD_TESTING=On -DBUILD_EXPERIMENTAL_BINDING=On -DVW_FEAT_CSV=On -DVW_FEAT_CB_GRAPH_FEEDBACK=On -DVW_INSTALL=Off
       - name: Build
         run: cmake --build build --target all
       - name: Unit tests

.github/workflows/build_windows_cmake.yml

@@ -55,7 +55,7 @@ jobs:
         run: ${{ env.VCPKG_ROOT }}/vcpkg.exe --triplet x64-windows install zlib flatbuffers
       - name: Generate project files
         run: |
-          cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64" -DVCPKG_MANIFEST_MODE=OFF -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On -Dvw_BUILD_NET_FRAMEWORK=On
+          cmake -S "${{ env.SOURCE_DIR }}" -B "${{ env.CMAKE_BUILD_DIR }}" -A "x64" -DVCPKG_MANIFEST_MODE=OFF -DCMAKE_TOOLCHAIN_FILE="${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake" -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On -DVW_FEAT_CB_GRAPH_FEEDBACK=On -Dvw_BUILD_NET_FRAMEWORK=On
       - name: Build project
         run: |
           cmake --build "${{ env.CMAKE_BUILD_DIR }}" --config ${{ matrix.build_config }} --verbose

.github/workflows/valgrind.yml

@@ -21,7 +21,7 @@ jobs:
           submodules: recursive
       - name: Build C++ VW binary
         run: |
-          cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_EXPERIMENTAL_BINDING=On -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On
+          cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DBUILD_EXPERIMENTAL_BINDING=On -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On -DVW_FEAT_CB_GRAPH_FEEDBACK=On
           cmake --build build
       - name: Upload vw binary
         uses: actions/upload-artifact@v2

.github/workflows/vendor_build.yml

@@ -39,6 +39,7 @@ jobs:
           -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
           -DVW_FEAT_FLATBUFFERS=Off
           -DVW_FEAT_CSV=On
+          -DVW_FEAT_CB_GRAPH_FEEDBACK=On
           -DRAPIDJSON_SYS_DEP=Off
           -DFMT_SYS_DEP=Off
           -DSPDLOG_SYS_DEP=Off
@@ -77,6 +78,7 @@ jobs:
           -DUSE_LATEST_STD=On
           -DVW_FEAT_FLATBUFFERS=Off
           -DVW_FEAT_CSV=On
+          -DVW_FEAT_CB_GRAPH_FEEDBACK=On
           -DRAPIDJSON_SYS_DEP=Off
           -DFMT_SYS_DEP=Off
           -DSPDLOG_SYS_DEP=Off
@@ -109,6 +111,7 @@ jobs:
           -DCMAKE_BUILD_TYPE=${{matrix.build_type}}
           -DVW_FEAT_FLATBUFFERS=Off
           -DVW_FEAT_CSV=On
+          -DVW_FEAT_CB_GRAPH_FEEDBACK=On
           -DRAPIDJSON_SYS_DEP=Off
           -DFMT_SYS_DEP=Off
           -DSPDLOG_SYS_DEP=Off

.gitmodules

@@ -22,3 +22,9 @@
 [submodule "ext_libs/sse2neon"]
 	path = ext_libs/sse2neon/sse2neon
 	url = https://github.com/DLTcollab/sse2neon
+[submodule "ext_libs/ensmallen"]
+	path = ext_libs/ensmallen
+	url = https://github.com/mlpack/ensmallen.git
+[submodule "ext_libs/armadillo-code"]
+	path = ext_libs/armadillo-code
+	url = https://gitlab.com/conradsnicta/armadillo-code.git

.scripts/build.cmd

@@ -18,6 +18,7 @@ cmake -S "%vwRoot%" -B "%vwRoot%\build" -G "Visual Studio 16 2019" -A "x64" ^
     -Dvw_BUILD_NET_FRAMEWORK=On ^
     -DVW_FEAT_FLATBUFFERS=On ^
     -DVW_FEAT_CSV=On ^
+    -DVW_FEAT_CB_GRAPH_FEEDBACK=On ^
     -Dvw_BUILD_NET_FRAMEWORK=On ^
     -DRAPIDJSON_SYS_DEP=Off ^
     -DFMT_SYS_DEP=Off ^

.scripts/linux/build-static-java.sh

@@ -10,7 +10,7 @@ mkdir -p build
 cd build
 # Boost unit tests don't like the static linking
 # /usr/local/bin/gcc + g++ is 9.2.0 version
-cmake -E env LDFLAGS="-Wl,--exclude-libs,ALL -static-libgcc -static-libstdc++" cmake .. -DCMAKE_BUILD_TYPE=Release -DWARNINGS=Off -DBUILD_JAVA=On -DBUILD_DOCS=Off -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On\
+cmake -E env LDFLAGS="-Wl,--exclude-libs,ALL -static-libgcc -static-libstdc++" cmake .. -DCMAKE_BUILD_TYPE=Release -DWARNINGS=Off -DBUILD_JAVA=On -DBUILD_DOCS=Off -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On -DVW_FEAT_CB_GRAPH_FEEDBACK=On\
  -DBUILD_PYTHON=Off -DSTATIC_LINK_VW_JAVA=On -DCMAKE_C_COMPILER=/usr/local/bin/gcc -DCMAKE_CXX_COMPILER=/usr/local/bin/g++ \
  -DBUILD_TESTING=Off -DVW_ZLIB_SYS_DEP=Off -DBUILD_SHARED_LIBS=Off -DVW_BUILD_LAS_WITH_SIMD=Off
 NUM_PROCESSORS=$(nproc)

.scripts/linux/build-with-coverage.sh

@@ -6,5 +6,5 @@ SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
 REPO_DIR=$SCRIPT_DIR/../../
 cd $REPO_DIR
 
-cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DVW_GCOV=ON -DWARNINGS=OFF -DBUILD_JAVA=Off -DBUILD_PYTHON=Off -DBUILD_TESTING=On -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On
+cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=Debug -DVW_GCOV=ON -DWARNINGS=OFF -DBUILD_JAVA=Off -DBUILD_PYTHON=Off -DBUILD_TESTING=On -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On -DVW_FEAT_CB_GRAPH_FEEDBACK=On
 cmake --build build

.scripts/linux/build.sh

@@ -9,5 +9,5 @@ cd $REPO_DIR
 # If parameter 1 is not supplied, it defaults to Release
 BUILD_CONFIGURATION=${1:-Release}
 
-cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=${BUILD_CONFIGURATION} -DWARNINGS=Off -DWARNING_AS_ERROR=On -DVW_BUILD_VW_C_WRAPPER=Off -DBUILD_JAVA=On -DBUILD_PYTHON=Off -DBUILD_TESTING=On -DBUILD_EXPERIMENTAL_BINDING=On -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On
+cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=${BUILD_CONFIGURATION} -DWARNINGS=Off -DWARNING_AS_ERROR=On -DVW_BUILD_VW_C_WRAPPER=Off -DBUILD_JAVA=On -DBUILD_PYTHON=Off -DBUILD_TESTING=On -DBUILD_EXPERIMENTAL_BINDING=On -DVW_FEAT_FLATBUFFERS=On -DVW_FEAT_CSV=On -DVW_FEAT_CB_GRAPH_FEEDBACK=On
 cmake --build build --target all

CMakePresets.json

@@ -88,6 +88,10 @@
         "VW_FEAT_CSV": {
           "type": "BOOL",
           "value": "On"
+        },
+        "VW_FEAT_CB_GRAPH_FEEDBACK": {
+          "type": "BOOL",
+          "value": "On"
         }
       }
     },

ThirdPartyNotices.txt

@@ -246,4 +246,79 @@ FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 DEALINGS IN THE SOFTWARE.
 =========================================
-anthonywilliams/ccia_code_samples NOTICES AND INFORMATION
+anthonywilliams/ccia_code_samples NOTICES AND INFORMATION
+
+Armadillo NOTICES AND INFORMATION BEGIN HERE
+=========================================
+Armadillo C++ Linear Algebra Library
+Copyright 2008-2023 Conrad Sanderson (https://conradsanderson.id.au)
+Copyright 2008-2016 National ICT Australia (NICTA)
+Copyright 2017-2023 Data61 / CSIRO
+
+This product includes software developed by Conrad Sanderson (https://conradsanderson.id.au)
+This product includes software developed at National ICT Australia (NICTA)
+This product includes software developed at Data61 / CSIRO
+
+---
+
+Attribution Notice.
+As per UN General Assembly Resolution A/RES/ES-11/1
+adopted on 2 March 2022 with 141 votes in favour and 5 votes against,
+we attribute the violation of the sovereignty and territorial integrity of Ukraine,
+and subsequent destruction of many Ukrainian cities and civilian infrastructure,
+to large-scale military aggression by the Russian Federation (aided by Belarus).
+Further details:
+https://undocs.org/A/RES/ES-11/1
+https://digitallibrary.un.org/record/3965290/files/A_RES_ES-11_1-EN.pdf
+https://digitallibrary.un.org/record/3965290/files/A_RES_ES-11_1-RU.pdf
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+=========================================
+Armadillo NOTICES AND INFORMATION
+
+Ensmallen NOTICES AND INFORMATION BEGIN HERE
+=========================================
+ensmallen is provided without any warranty of fitness for any purpose.  You
+can redistribute the library and/or modify it under the terms of the 3-clause
+BSD license.  The text of the 3-clause BSD license is contained below.
+
+----
+Copyright (c) 2011-2018, mlpack and ensmallen contributors (see COPYRIGHT.txt)
+All rights reserved.
+
+Redistribution and use of ensmallen in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+may be used to endorse or promote products derived from this software without
+specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+=========================================
+Ensmallen NOTICES AND INFORMATION

cmake/VowpalWabbitFeatures.cmake

@@ -9,10 +9,11 @@
 #   - The cmake variable VW_FEAT_X is set to ON, otherwise it is OFF
 #   - The C++ macro VW_FEAT_X_ENABLED is defined if the feature is enabled, otherwise it is not defined
 
-set(VW_ALL_FEATURES "CSV;FLATBUFFERS;LDA")
+set(VW_ALL_FEATURES "CSV;FLATBUFFERS;LDA;CB_GRAPH_FEEDBACK")
 
 option(VW_FEAT_FLATBUFFERS "Enable flatbuffers support" OFF)
 option(VW_FEAT_CSV "Enable csv parser" OFF)
+option(VW_FEAT_CB_GRAPH_FEEDBACK "Enable cb with graph feedback reduction" OFF)
 option(VW_FEAT_LDA "Enable lda reduction" ON)
 # option(VW_FEAT_LAS_SIMD "Enable large action space with explicit simd (only works with linux for now)" ON)
 

ext_libs/ext_libs.cmake

@@ -121,3 +121,10 @@ else()
   # header at the root of the repo rather than its own nested sse2neon/ dir
   target_include_directories(sse2neon SYSTEM INTERFACE "${CMAKE_CURRENT_LIST_DIR}/sse2neon")
 endif()
+
+if(VW_FEAT_CB_GRAPH_FEEDBACK)
+  add_library(mlpack_ensmallen INTERFACE)
+  target_include_directories(mlpack_ensmallen SYSTEM INTERFACE ${CMAKE_CURRENT_LIST_DIR}/armadillo-code/include)
+
+  target_include_directories(mlpack_ensmallen SYSTEM INTERFACE ${CMAKE_CURRENT_LIST_DIR}/ensmallen/include)
+endif()

test/train-sets/ref/help.stdout

@@ -620,6 +620,14 @@ Weight Options:
                                             uint, keep, necessary)
     --replay_m_count arg                    How many times (in expectation) should each example be played
                                             (default: 1 = permuting) (type: uint, default: 1)
+[Reduction] Experimental: Contextual Bandit Exploration with ADF with graph feedback Options:
+    --cb_explore_adf                        Online explore-exploit for a contextual bandit problem with multiline
+                                            action dependent features (type: bool, keep, necessary)
+    --gamma_scale arg                       Sets CB with graph feedback gamma parameter to gamma=[gamma_scale]*[num
+                                            examples]^1/2 (type: float, default: 1, keep)
+    --gamma_exponent arg                    Exponent on [num examples] in CB with graph feedback parameter
+                                            gamma (type: float, default: 0.5, keep)
+    --graph_feedback                        Graph feedback pdf (type: bool, keep, necessary, experimental)
 [Reduction] Experimental: Contextual Bandit Exploration with ADF with large action space filtering Options:
     --cb_explore_adf                        Online explore-exploit for a contextual bandit problem with multiline
                                             action dependent features (type: bool, keep, necessary)

vowpalwabbit/core/CMakeLists.txt

@@ -364,6 +364,11 @@ if(VW_FEAT_LDA)
   list(APPEND vw_core_sources src/reductions/lda_core.cc)
 endif()
 
+if(VW_FEAT_CB_GRAPH_FEEDBACK)
+  list(APPEND vw_core_headers include/vw/core/reductions/cb/cb_explore_adf_graph_feedback.h)
+  list(APPEND vw_core_sources src/reductions/cb/cb_explore_adf_graph_feedback.cc)
+endif()
+
 vw_add_library(
     NAME "core"
     TYPE "STATIC_ONLY"
@@ -384,6 +389,10 @@ if(VW_FEAT_LDA)
   target_link_libraries(vw_core PRIVATE $<BUILD_INTERFACE:${boost_math_target}>)
 endif()
 
+if(VW_FEAT_CB_GRAPH_FEEDBACK)
+  target_link_libraries(vw_core PRIVATE $<BUILD_INTERFACE:mlpack_ensmallen>)
+endif()
+
 target_include_directories(vw_core PRIVATE ${CMAKE_CURRENT_LIST_DIR}/src)
 
 if (VW_BUILD_LAS_WITH_SIMD AND (UNIX AND NOT APPLE) AND (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64"))
@@ -427,10 +436,7 @@ if(WIN32)
   target_compile_definitions(vw_core PUBLIC __SSE2__)
 endif()
 
-vw_add_test_executable(
-    FOR_LIB "core"
-    EXTRA_DEPS vw_test_common
-    SOURCES
+set(vw_core_test_sources
       tests/automl_test.cc
       tests/automl_weights_test.cc
       tests/baseline_cb_test.cc
@@ -496,6 +502,16 @@ vw_add_test_executable(
       tests/igl_test.cc
 )
 
+if(VW_FEAT_CB_GRAPH_FEEDBACK)
+  list(APPEND vw_core_test_sources tests/cb_graph_feedback_test.cc)
+endif()
+
+vw_add_test_executable(
+    FOR_LIB "core"
+    EXTRA_DEPS vw_test_common
+    SOURCES ${vw_core_test_sources}
+)
+
 if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME AND BUILD_TESTING)
   # Tests are allowed to access private headers.
   target_include_directories(vw_core_test PRIVATE $<TARGET_PROPERTY:vw_core,INCLUDE_DIRECTORIES>)

vowpalwabbit/core/include/vw/core/reductions/cb/cb_explore_adf_graph_feedback.h

@@ -0,0 +1,16 @@
+// Copyright (c) by respective owners including Yahoo!, Microsoft, and
+// individual contributors. All rights reserved. Released under a BSD (revised)
+// license as described in the file LICENSE.
+#pragma once
+
+#include "vw/core/vw_fwd.h"
+
+#include <memory>
+
+namespace VW
+{
+namespace reductions
+{
+std::shared_ptr<VW::LEARNER::learner> cb_explore_adf_graph_feedback_setup(VW::setup_base_i& stack_builder);
+}  // namespace reductions
+}  // namespace VW

vowpalwabbit/core/src/reduction_stack.cc

@@ -31,6 +31,9 @@
 #include "vw/core/reductions/cb/cb_explore_adf_bag.h"
 #include "vw/core/reductions/cb/cb_explore_adf_cover.h"
 #include "vw/core/reductions/cb/cb_explore_adf_first.h"
+#ifdef VW_FEAT_CB_GRAPH_FEEDBACK_ENABLED
+#  include "vw/core/reductions/cb/cb_explore_adf_graph_feedback.h"
+#endif
 #include "vw/core/reductions/cb/cb_explore_adf_greedy.h"
 #include "vw/core/reductions/cb/cb_explore_adf_large_action_space.h"
 #include "vw/core/reductions/cb/cb_explore_adf_regcb.h"
@@ -199,6 +202,9 @@ void prepare_reductions(std::vector<std::tuple<std::string, VW::reduction_setup_
   reductions.push_back(VW::reductions::automl_setup);
   reductions.push_back(VW::reductions::cb_explore_setup);
   reductions.push_back(VW::reductions::cb_explore_adf_large_action_space_setup);
+#ifdef VW_FEAT_CB_GRAPH_FEEDBACK_ENABLED
+  reductions.push_back(VW::reductions::cb_explore_adf_graph_feedback_setup);
+#endif
   reductions.push_back(VW::reductions::cb_explore_adf_greedy_setup);
   reductions.push_back(VW::reductions::cb_explore_adf_softmax_setup);
   reductions.push_back(VW::reductions::cb_explore_adf_rnd_setup);