ZOOKEEPER-1112: Add (Cyrus) SASL authentication support to C client library

This is a "respin" of apache#1054, which I withdrew due to some annoying shortcomings.

This changeset allows C clients to use SASL to authenticate with the ZooKeeper server.  It is loosely based on patches #1 and apache#2 by Tom Klonikowski, at https://reviews.apache.org/r/2252/, but the result has been extensively reworked to follow the semantics of the Java client:

  * No SASL operations are exposed through the API;

  * The configuration is provided, and stored, at "handle init time";

  * SASL authentication is automatically performed after each (re)connect.

It introduces an optional dependency on the Cyrus SASL library, which can either be autodetected (default) or configured using the `--without-sasl`/`--with-sasl[=DIR]` flags, or -DWITH_CYRUS_SASL for CMake/Windows.

`TestServerRequireClientSASLAuth.cc` has been renamed to `TestSASLAuth.cc`, and a test has been added which successfully (re)authenticates using the `DIGEST-MD5` mechanism.  The code has also been used to successfully authenticate clients via `GSSAPI`/Kerberos.

This commit also adds SASL support to the `cli.c` client.

Co-authored-by: Tom Klonikowski <klonik_tinformatik.haw-hamburg.de>

Author: Damien Diederen <[email protected]>

Reviewers: Mate Szalay-Beko <[email protected]>, Norbert Kalmar <[email protected]>

Closes apache#1134 from ztzg/ZOOKEEPER-1112-c-client-sasl-support-v2

Author: ztzg

README_packaging.md

@@ -9,14 +9,15 @@ http://bigtop.apache.org/
 ## Requirements
 
 - you need maven to build the java code
-- gcc, cppunit, openssl and python-setuptools are required to build C and python bindings. (only needed when using `-Pfull-build`)
+- gcc, cppunit, openssl and python-setuptools are required to build C and python bindings (only needed when using `-Pfull-build`).  Cyrus SASL is optional, but recommended for a maximally functional client.
 
 On RHEL machine:
 
 ```
 yum install cppunit
 yum install python-setuptools
 yum install openssl openssl-devel
+yum install cyrus-sasl-md5 cyrus-sasl-gssapi cyrus-sasl-devel
 ```
 
 On Ubuntu:
@@ -25,6 +26,7 @@ On Ubuntu:
 apt-get install cppunit
 apt-get install python-setuptools
 apt-get install openssl libssl-dev
+apt-get install libsasl2-modules-gssapi-mit libsasl2-modules libsasl2-dev
 ```
 
 
@@ -63,7 +65,9 @@ Optional parameters you might consider when using maven:
                               Use `-Dc-client-openssl=no` to explicitly disable SSL feature in C client. Or use 
                               `-Dc-client-openssl=/path/to/openssl/` if you want to use a non-default / specific 
                               openssl library location.
-    
+-  `-Dc-client-sasl`      -   specify SASL support and Cyrus SASL 1.x library location.  Works similarly to the
+                              `c-client-openssl` flag above (`yes`, `no`, or path).
+
 Please note: if you don't provide the `-Pfull-build` parameter, then the C client will not be built, the C client tests
 will not be executed and the previous C client builds will no be cleaned up (e.g. with simply using `mvn clean`).
 

dev/docker/Dockerfile

@@ -20,4 +20,4 @@
 FROM maven:3.6.3-jdk-8
 
 RUN apt-get update
-RUN apt-get install -y g++ cmake autoconf libcppunit-dev libtool openssl libssl-dev
+RUN apt-get install -y g++ cmake autoconf libcppunit-dev libtool openssl libssl-dev libsasl2-modules-gssapi-mit libsasl2-modules libsasl2-dev

pom.xml

@@ -339,8 +339,9 @@
     <spotbugsannotations.version>3.1.9</spotbugsannotations.version>
     <checkstyle.version>8.17</checkstyle.version>
 
-    <!-- parameter to pass to C client build -->
+    <!-- parameters to pass to C client build -->
     <c-client-openssl>yes</c-client-openssl>
+    <c-client-sasl>yes</c-client-sasl>
 
   </properties>
 

tools/cmake/Modules/FindCyrusSASL.cmake

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# - Find Cyrus SASL (sasl.h, libsasl2.so)
+#
+# This module defines
+#  CYRUS_SASL_INCLUDE_DIR, directory containing headers
+#  CYRUS_SASL_SHARED_LIB, path to Cyrus SASL's shared library
+#  CYRUS_SASL_FOUND, whether Cyrus SASL and its plugins have been found
+#
+# It also defines the following IMPORTED targets:
+#  CyrusSASL
+#
+# Hints:
+#  Set CYRUS_SASL_ROOT_DIR to the root directory of a Cyrus SASL installation.
+#
+# The initial version of this file was extracted from
+# https://github.com/cloudera/kudu, at the following commit:
+#
+#  commit 9806863e78107505a622b44112a897189d9b3c24
+#  Author: Dan Burkert <[email protected]>
+#  Date:   Mon Nov 30 12:15:36 2015 -0800
+#
+#      Enable C++11
+
+find_path(CYRUS_SASL_INCLUDE_DIR sasl/sasl.h HINTS "${CYRUS_SASL_ROOT_DIR}/include")
+find_library(CYRUS_SASL_SHARED_LIB sasl2 HINTS "${CYRUS_SASL_ROOT_DIR}/lib")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(CYRUS_SASL REQUIRED_VARS
+  CYRUS_SASL_SHARED_LIB CYRUS_SASL_INCLUDE_DIR)
+
+if(CYRUS_SASL_FOUND)
+  if(NOT TARGET CyrusSASL)
+    add_library(CyrusSASL UNKNOWN IMPORTED)
+    set_target_properties(CyrusSASL PROPERTIES
+      INTERFACE_INCLUDE_DIRECTORIES "${CYRUS_SASL_INCLUDE_DIR}"
+      IMPORTED_LINK_INTERFACE_LANGUAGES "C"
+      IMPORTED_LOCATION "${CYRUS_SASL_SHARED_LIB}")
+  endif()
+endif()

zookeeper-client/zookeeper-client-c/CMakeLists.txt

@@ -20,6 +20,8 @@ project(zookeeper VERSION 3.7.0)
 set(email [email protected])
 set(description "zookeeper C client")
 
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/../../tools/cmake/Modules")
+
 # general options
 if(UNIX)
   add_compile_options(-Wall -fPIC)
@@ -61,6 +63,20 @@ if(WANT_SOCK_CLOEXEC AND HAVE_SOCK_CLOEXEC)
   set(SOCK_CLOEXEC_ENABLED 1)
 endif()
 
+# Cyrus SASL 2.x
+option(WITH_CYRUS_SASL "turn ON/OFF Cyrus SASL 2.x support, or define SASL library location (default: ON)" ON)
+message("-- using WITH_CYRUS_SASL=${WITH_CYRUS_SASL}")
+if(NOT WITH_CYRUS_SASL STREQUAL "OFF")
+  if(NOT WITH_CYRUS_SASL STREQUAL "ON")
+    set(CYRUS_SASL_ROOT_DIR "${WITH_CYRUS_SASL}")
+  endif()
+  find_package(CyrusSASL)
+  if(CYRUS_SASL_FOUND)
+    message("-- Cyrus SASL 2.x found! will build with SASL support.")
+  else()
+    message("-- WARNING: unable to find Cyrus SASL 2.x! will build without SASL support.")
+  endif()
+endif()
 
 # The function `to_have(in out)` converts a header name like `arpa/inet.h`
 # into an Autotools style preprocessor definition `HAVE_ARPA_INET_H`.
@@ -171,6 +187,10 @@ else()
   list(APPEND zookeeper_sources src/st_adaptor.c)
 endif()
 
+if(CYRUS_SASL_FOUND)
+  list(APPEND zookeeper_sources src/zk_sasl.c)
+endif()
+
 if(WIN32)
   list(APPEND zookeeper_sources src/winport.c)
 endif()
@@ -203,6 +223,11 @@ if(WANT_SYNCAPI AND NOT WIN32)
   target_link_libraries(zookeeper PUBLIC Threads::Threads)
 endif()
 
+if(CYRUS_SASL_FOUND)
+  target_compile_definitions(zookeeper PUBLIC HAVE_CYRUS_SASL_H)
+  target_link_libraries(zookeeper PUBLIC CyrusSASL)
+endif()
+
 # cli executable
 add_executable(cli src/cli.c)
 target_link_libraries(cli zookeeper)

zookeeper-client/zookeeper-client-c/LICENSE

@@ -333,3 +333,53 @@
  * copied and put under another distribution licence
  * [including the GNU Public Licence.]
  */
+
+===========================================================================================
+===       The following part contains the license for the Cyrus SASL 2.x library        ===
+===                           used for optional SASL support                            ===
+===========================================================================================
+
+/* CMU libsasl
+ * Tim Martin
+ * Rob Earhart
+ * Rob Siemborski
+ */
+/* 
+ * Copyright (c) 1998-2003 Carnegie Mellon University.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer. 
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. The name "Carnegie Mellon University" must not be used to
+ *    endorse or promote products derived from this software without
+ *    prior written permission. For permission or any other legal
+ *    details, please contact  
+ *      Office of Technology Transfer
+ *      Carnegie Mellon University
+ *      5000 Forbes Avenue
+ *      Pittsburgh, PA  15213-3890
+ *      (412) 268-4387, fax: (412) 268-7395
+ *      [email protected]
+ *
+ * 4. Redistributions of any form whatsoever must retain the following
+ *    acknowledgment:
+ *    "This product includes software developed by Computing Services
+ *     at Carnegie Mellon University (http://www.cmu.edu/computing/)."
+ *
+ * CARNEGIE MELLON UNIVERSITY DISCLAIMS ALL WARRANTIES WITH REGARD TO
+ * THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS, IN NO EVENT SHALL CARNEGIE MELLON UNIVERSITY BE LIABLE
+ * FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN
+ * AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING
+ * OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */

zookeeper-client/zookeeper-client-c/Makefile.am

@@ -13,10 +13,16 @@ if WANT_OPENSSL
   OPENSSL_LIB_LDFLAGS = -lssl -lcrypto
 endif
 
-AM_CPPFLAGS = -I${srcdir}/include -I${srcdir}/tests -I${srcdir}/generated $(SOLARIS_CPPFLAGS) $(OPENSSL_CPPFLAGS)
+if WANT_SASL
+  SASL_CPPFLAGS = -DHAVE_CYRUS_SASL_H
+  SASL_LIB_LDFLAGS = -lsasl2
+  SASL_SRC = src/zk_sasl.c
+endif
+
+AM_CPPFLAGS = -I${srcdir}/include -I${srcdir}/tests -I${srcdir}/generated $(SOLARIS_CPPFLAGS) $(OPENSSL_CPPFLAGS) $(SASL_CPPFLAGS)
 AM_CFLAGS = -Wall -Werror -Wdeclaration-after-statement
 AM_CXXFLAGS = -Wall $(USEIPV6)
-LIB_LDFLAGS = -no-undefined -version-info 2 $(SOLARIS_LIB_LDFLAGS) $(OPENSSL_LIB_LDFLAGS)
+LIB_LDFLAGS = -no-undefined -version-info 2 $(SOLARIS_LIB_LDFLAGS) $(OPENSSL_LIB_LDFLAGS) $(SASL_LIB_LDFLAGS)
 
 # Additional flags for coverage testing (if enabled)
 if ENABLEGCOV
@@ -37,7 +43,7 @@ COMMON_SRC = src/zookeeper.c include/zookeeper.h include/zookeeper_version.h inc
     src/recordio.c include/recordio.h include/proto.h \
     src/zk_adaptor.h generated/zookeeper.jute.c \
     src/zk_log.c src/zk_hashtable.h src/zk_hashtable.c \
-	src/addrvec.h src/addrvec.c
+    src/addrvec.h src/addrvec.c $(SASL_SRC)
 
 # These are the symbols (classes, mostly) we want to export from our library.
 EXPORT_SYMBOLS = '(zoo_|zookeeper_|zhandle|Z|format_log_message|log_message|logLevel|deallocate_|allocate_|zerror|is_unrecoverable)'
@@ -67,13 +73,13 @@ endif
 bin_PROGRAMS = cli_st
 
 cli_st_SOURCES = src/cli.c
-cli_st_LDADD = libzookeeper_st.la
+cli_st_LDADD = libzookeeper_st.la $(SASL_LIB_LDFLAGS)
 
 if WANT_SYNCAPI
 bin_PROGRAMS += cli_mt load_gen
 
 cli_mt_SOURCES = src/cli.c
-cli_mt_LDADD = libzookeeper_mt.la
+cli_mt_LDADD = libzookeeper_mt.la $(SASL_LIB_LDFLAGS)
 cli_mt_CFLAGS = -DTHREADED
 
 load_gen_SOURCES = src/load_gen.c
@@ -113,7 +119,7 @@ TEST_SOURCES = \
 	tests/ZooKeeperQuorumServer.h \
 	tests/TestReadOnlyClient.cc \
 	tests/TestLogClientEnv.cc \
-    tests/TestServerRequireClientSASLAuth.cc \
+        tests/TestSASLAuth.cc \
 	$(NULL)
 
 if SOLARIS
@@ -127,14 +133,14 @@ check_PROGRAMS = zktest-st
 TESTS_ENVIRONMENT = ZKROOT=${srcdir}/../.. \
                     CLASSPATH=$$CLASSPATH:$$CLOVER_HOME/lib/clover*.jar
 nodist_zktest_st_SOURCES = $(TEST_SOURCES)
-zktest_st_LDADD = libzkst.la libhashtable.la $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) -ldl
+zktest_st_LDADD = libzkst.la libhashtable.la $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) $(SASL_LIB_LDFLAGS) -ldl
 zktest_st_CXXFLAGS = -DUSE_STATIC_LIB $(CPPUNIT_CFLAGS) $(USEIPV6) $(SOLARIS_CPPFLAGS)
 zktest_st_LDFLAGS = -shared $(SYMBOL_WRAPPERS) $(SOLARIS_LIB_LDFLAGS)
 
 if WANT_SYNCAPI
   check_PROGRAMS += zktest-mt
   nodist_zktest_mt_SOURCES = $(TEST_SOURCES) tests/PthreadMocks.cc
-  zktest_mt_LDADD = libzkmt.la libhashtable.la -lpthread $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) -ldl
+  zktest_mt_LDADD = libzkmt.la libhashtable.la -lpthread $(CPPUNIT_LIBS) $(OPENSSL_LIB_LDFLAGS) $(SASL_LIB_LDFLAGS) -ldl
   zktest_mt_CXXFLAGS = -DUSE_STATIC_LIB -DTHREADED $(CPPUNIT_CFLAGS) $(USEIPV6)
 if SOLARIS
   SHELL_SYMBOL_WRAPPERS_MT = cat ${srcdir}/tests/wrappers-mt.opt

zookeeper-client/zookeeper-client-c/README

@@ -83,6 +83,8 @@ Follow steps 1 and 2 above, and then continue here.
    -DWANT_CPPUNIT      ON except on Windows, OFF disables the tests
    -DWITH_OPENSSL      ON by default, OFF disables the SSL support. You can also
                        specify a custom path by -DWITH_OPENSSL=/path/to/openssl/
+   -DWITH_CYRUS_SASL   ON by default, OFF disables SASL support. You can also
+                       specify a custom path by -DWITH_CYRUS_SASL=/path/to/cyrus-sasl/
    -DBUILD_SHARED_LIBS not yet supported, only static libraries are built
    other CMake options see "cmake --help" for generic options, such as generator
 

zookeeper-client/zookeeper-client-c/configure.ac

@@ -143,6 +143,36 @@ fi
 
 AM_CONDITIONAL([WANT_SYNCAPI],[test "x$with_syncapi" != xno])
 
+dnl Cyrus SASL 2.x
+AC_ARG_WITH(sasl,
+ [AC_HELP_STRING([--with-sasl[=DIR]], [build with SASL support via Cyrus SASL 2.x (default=auto)])],
+ [], [with_sasl=yes])
+if test "x$with_sasl" != "xno"; then
+    saved_CPPFLAGS="$CPPFLAGS"
+    saved_LDFLAGS="$LDFLAGS"
+    if test "x$with_sasl" != "xyes" ; then
+        CPPFLAGS="$CPPFLAGS -I$with_sasl/include"
+        LDFLAGS="$LDFLAGS -L$with_sasl/lib"
+    fi
+    have_sasl=no
+    AC_CHECK_HEADER(sasl/sasl.h, [
+     AC_CHECK_LIB(sasl2, sasl_client_init, [have_sasl=yes])])
+    if test "x$have_sasl" != "xyes"; then
+        CPPFLAGS="$saved_CPPFLAGS"
+        LDFLAGS="$saved_LDFLAGS"
+    fi
+fi
+if test "x$with_sasl" != xno && test "x$have_sasl" = xno; then
+    AC_MSG_WARN([cannot build SASL support -- sasl2 not found])
+    with_sasl=no
+fi
+if test "x$with_sasl" != xno; then
+    AC_MSG_NOTICE([building with SASL support])
+else
+    AC_MSG_NOTICE([building without SASL support])
+fi
+AM_CONDITIONAL([WANT_SASL],[test "x$with_sasl" != xno])
+
 # Checks for header files.
 AC_HEADER_STDC
 AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/socket.h sys/time.h unistd.h sys/utsname.h])

zookeeper-client/zookeeper-client-c/include/proto.h

@@ -46,6 +46,7 @@ extern "C" {
 #define ZOO_CLOSE_OP -11
 #define ZOO_SETAUTH_OP 100
 #define ZOO_SETWATCHES_OP 101
+#define ZOO_SASL_OP 102
 
 #ifdef __cplusplus
 }