diff --git a/README.md b/README.md new file mode 100644 index 0000000..173d2f7 --- /dev/null +++ b/README.md @@ -0,0 +1,89 @@ +# Franzy-Nippy + +[Kafka](http://kafka.apache.org/documentation.html) serializer using the excellent serialization library, [Nippy](https://github.com/ptaoussanis/nippy). + +Great fit with [Franzy](https://github.com/ymilky/franzy), a Clojure Kafka client, though not required. Feel free to use this serializer with any Kafka client, including via Java, Scala, Groovy, or any other JVM language. + +## Why + +* You want to compress your data when sending to and from Kafka. +* You want fast de/serialization, via nippy and its underlying implementation. +* You are using a Kafka client such as [Franzy](https://github.com/ymilky/franzy) and need a pluggable, robust serializer. +* You want to serialize Clojure data types with little effort. +* You want seamless serialization, no embedded serialization calls at call sites or `.getBytes` ugly things floating around. + +## Docs + +* Read the browsable [API](http://ymilky.github.io/franzy-nippy/api/index.html) +* See [Franzy Source](https://github.com/ymilky/franzy) and docs for more information about serializers/deserializers. +* For more information about serializer options, compression, etc, see the official [Nippy](https://github.com/ptaoussanis/nippy) repo. + +## Installation + +Add the necessary dependency to your project: + +```clojure +[ymilky/franzy-nippy "0.0.1"] +``` +[![Clojars Project](https://img.shields.io/clojars/v/ymilky/franzy-fressian.svg)](https://clojars.org/ymilky/franzy-fressian) + +## Serializing + +First, require: + +```clojure +(ns my-ns + (:require [franzy.serialization.nippy.serializers :as serializers])) +``` + +Then use with a producer, such as the one with [Franzy](https://github.com/ymilky/franzy). + +```clojure + (let [;;optionally specify via Kakfa Config key - value.serializer using fully qualified class name + pc {:bootstrap.servers ["127.0.0.1"]} + ;;Serializes producer record keys, ex: (keyword-serializer) from Franzy + key-serializer (your-key-serializer-type) + ;;Serializes producer record values using nippy, call (serializers/nippy-serializer options) to pass nippy options + value-serializer (serializers/nippy-serializer)] + (with-open [p (producer/make-producer pc key-serializer value-serializer)] + ;;spray useless data to Kafka using Clojure types + (send-async! "aliens-wearing-curtains" 2262 + ["s: Who? The Narn or the Centauri? k: yes" "...is dead. + k: We are all Kosh." "If you watch Legend of the Rangers, you will die."])))) +``` + +## Deserializing + +First, require: + +```clojure +(ns my-ns + (:require [franzy.serialization.nippy.deserializers :as deserializers])) +``` + +Then use with a consumer, such as the one with [Franzy](https://github.com/ymilky/franzy). + +```clojure + (let [;;optionally specify via Kafka Config key - value.deserializer using fully qualified class name + cc {:bootstrap.servers ["127.0.0.1:9092"] + :group.id "we-are-purple"} + ;;Deserializes your record keys, ex: (keyword-deserializer) from Franzy + key-deserializer (your-key-deserializer-type) + ;;the value deserializer should be the same kind as the serializer, don't mix and match + ;;call (serializers/nippy-deserializer options) to pass nippy options + value-deserializer (deserializers/nippy-deserializer) + topic "aliens-wearing-curtains" + topic-partitions [{:topic topic :partition 2262}]] + (with-open [c (consumer/make-consumer cc key-deserializer value-deserializer)] + (assign-partitions! c topic-partitions) + (seek-to-beginning-offset! c topic-partitions) + (poll! c))) +``` + +## License + +Copyright © 2016 Yossi M. (ymilky). + +Distributed under the Eclipse Public License either version 1.0 or (at your option) any later version. + +Use at your own risk, I am not responsible or liable. Please give credit if you use pieces of this library or otherwise, it is much appreciated. diff --git a/project.clj b/project.clj new file mode 100644 index 0000000..67e156a --- /dev/null +++ b/project.clj @@ -0,0 +1,30 @@ +(defproject ymilky/franzy-nippy "0.0.1" + :description "A Kafka Serializer/Deserializer supporting Nippy, and an add-on for Franzy, a Clojure Kafka client." + :url "https://github.com/ymilky/franzy-nippy" + :author "ymilky and others, but see README" + :license {:name "Eclipse Public License" + :url "http://www.eclipse.org/legal/epl-v10.html"} + :repositories {"snapshots" {:url "https://clojars.org/repo" + :username :env + :password :env + :sign-releases false} + "releases" {:url "https://clojars.org/repo" + :username :env + :password :env + :sign-releases false}} + :dependencies [[org.clojure/clojure "1.8.0"] + [org.apache.kafka/kafka-clients "0.9.0.1"] + [com.taoensso/nippy "2.11.1"]] + :plugins [[lein-codox "0.9.4"]] + :codox {:metadata {:doc/format :markdown} + :doc-paths ["README.md"] + :output-path "doc/api"} + :profiles {:dev {:dependencies [[midje "1.7.0"]] + :plugins [[lein-midje "3.2"] + [lein-set-version "0.4.1"] + [lein-update-dependency "0.1.2"] + [lein-pprint "1.1.1"]]} + :reflection-check {:global-vars + {*warn-on-reflection* true + *assert* false + *unchecked-math* :warn-on-boxed}}}) diff --git a/src/franzy/serialization/nippy/deserializers.clj b/src/franzy/serialization/nippy/deserializers.clj new file mode 100644 index 0000000..a862b48 --- /dev/null +++ b/src/franzy/serialization/nippy/deserializers.clj @@ -0,0 +1,20 @@ +(ns franzy.serialization.nippy.deserializers + (:require [taoensso.nippy :as nippy]) + (:import (org.apache.kafka.common.serialization Deserializer))) + +(deftype NippyDeserializer [opts] + Deserializer + (configure [_ _ _]) + (deserialize [_ _ data] + (nippy/thaw data opts)) + (close [_])) + +(defn nippy-deserializer + "Nippy deserializer for Apache Kafka. + Use for serializing Kafka values. + + > Notes: You may pass any of the built-in nippy options via the opts map, using + the 1-arity version of this function." + (^NippyDeserializer [] (nippy-deserializer nil)) + (^NippyDeserializer [opts] + (NippyDeserializer. opts))) diff --git a/src/franzy/serialization/nippy/serializers.clj b/src/franzy/serialization/nippy/serializers.clj new file mode 100644 index 0000000..0e3f140 --- /dev/null +++ b/src/franzy/serialization/nippy/serializers.clj @@ -0,0 +1,20 @@ +(ns franzy.serialization.nippy.serializers + (:require [taoensso.nippy :as nippy]) + (:import (org.apache.kafka.common.serialization Serializer))) + +(deftype NippySerializer [opts] + Serializer + (configure [_ _ _]) + (serialize [_ _ data] + (nippy/freeze data opts)) + (close [_])) + +(defn nippy-serializer + "Nippy serializer for Apache Kafka. + Use for serializing Kafka values. + + > Notes: You may pass any of the built-in nippy options via the opts map, using + the 1-arity version of this function." + (^NippySerializer [] (nippy-serializer nil)) + (^NippySerializer [opts] + (NippySerializer. opts))) diff --git a/test/franzy/core_test.clj b/test/franzy/core_test.clj new file mode 100644 index 0000000..f58fb9e --- /dev/null +++ b/test/franzy/core_test.clj @@ -0,0 +1,3 @@ +(ns franzy.core-test + (:require [midje.sweet :refer :all] + [franzy-nippy.core :refer :all])) diff --git a/test/franzy/serialization/nippy/serialization_tests.clj b/test/franzy/serialization/nippy/serialization_tests.clj new file mode 100644 index 0000000..71413a6 --- /dev/null +++ b/test/franzy/serialization/nippy/serialization_tests.clj @@ -0,0 +1,105 @@ +(ns franzy.serialization.nippy.serialization-tests + (:require [midje.sweet :refer :all] + [franzy.serialization.nippy.serializers :as serializers] + [franzy.serialization.nippy.deserializers :as deserializers]) + (:import (org.apache.kafka.common.serialization Deserializer Serializer) + (java.util UUID))) + +(facts + "Nippy serializers/deserializers serialize and deserialize Clojure data structures." :serializers + (let [serializer (serializers/nippy-serializer) + deserializer (deserializers/nippy-deserializer) + topic "music-education" + data {:good-bands ["New Order" "Joy Division" "The Cure" "The Smiths" "Pulp" "Jesus and Mary Chain"] + :terrible-bands #{"The Eagles" "Most of American Music in the 90s"} + :things-pretending-to-be-bands `("Justin Bieber" "Kanye West" "Beonce" "Arcade Fire") + :good-year 1984 + :essential-album :script-of-the-bridge-by-the-chameleons + :most-overrated "Jennifer Lopez" + :good-music-this-year nil}] + (fact + "A nippy serializer is a Kafka serializer." + (instance? Serializer serializer) => true) + (fact + "A nippy deserializer is a Kafka deserializer." + (instance? Deserializer deserializer) => true) + (fact + "A nippy serializer can serialize a string." + (let [val "Slowdive is under appreciated"] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize an number." + (let [val 1982] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a short." + (let [val (short 1)] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a long." + (let [val (long Long/MAX_VALUE)] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize an integer." + (let [val (int Integer/MAX_VALUE)] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a UUID." + (let [val (UUID/randomUUID)] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a map." + (let [val {:great-song "porcelain raft - dragonfly"}] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a vector." + (let [val ["DIIV" "Chapterhouse" "Moose" "Ride"]] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a list." + (let [val '("Suede" "Blur" "Elbow" "Cast" "Doves")] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a set." + (let [val #{"Forget That You're Young" "Black Satin" "Hallucinations" "With My Eyes Closed" "Here Comes Mary"}] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a keywords." + (let [val :raveonettes] + (.serialize serializer topic val) =not=> nil + (->> (.serialize serializer topic val) + (.deserialize deserializer topic)) => val)) + (fact + "A nippy serializer can serialize a function, but it's stupid to do so." + (let [val (fn [x] (+ 1 x))] + (.serialize serializer topic val) =not=> nil)) + (fact + "A nippy serializer should be able to produce the same data in a round trip." :serializers + (->> (.serialize serializer topic data) + (.deserialize deserializer topic)) => data) + (fact + "A nippy serializer should handle nil data, just in case..." + (->> (.serialize serializer topic nil) =not=> nil)) + (fact + "A nippy deserializer should handle nil data, just in case..." + (->> (.deserialize deserializer topic nil) =not=> nil))))