diff --git a/conf/0.8/schema.json b/conf/0.8/schema.json index f98f69e..f597858 100755 --- a/conf/0.8/schema.json +++ b/conf/0.8/schema.json @@ -61,6 +61,9 @@ "SuperUUID":{ "subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType", "comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType", - "column_type":"Super"} + "column_type":"Super"}, + "CompositeColumnConversion":{ + "comparator_type":"org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type)", + "column_type":"Standard"} } } diff --git a/conf/0.8/schema.txt b/conf/0.8/schema.txt index e362fc6..a23f7c2 100755 --- a/conf/0.8/schema.txt +++ b/conf/0.8/schema.txt @@ -48,4 +48,4 @@ create keyspace TypeConversions with use TypeConversions; create column family UUIDColumnConversion with comparator = TimeUUIDType; create column family SuperUUID with comparator = TimeUUIDType and column_type = Super; - +create column family CompositeColumnConversion with comparator = 'CompositeType(IntegerType, UTF8Type)'; diff --git a/conf/1.0/schema.json b/conf/1.0/schema.json index f98f69e..f597858 100755 --- a/conf/1.0/schema.json +++ b/conf/1.0/schema.json @@ -61,6 +61,9 @@ "SuperUUID":{ "subcomparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType", "comparator_type":"org.apache.cassandra.db.marshal.TimeUUIDType", - "column_type":"Super"} + "column_type":"Super"}, + "CompositeColumnConversion":{ + "comparator_type":"org.apache.cassandra.db.marshal.CompositeType(org.apache.cassandra.db.marshal.IntegerType,org.apache.cassandra.db.marshal.UTF8Type)", + "column_type":"Standard"} } } diff --git a/conf/1.0/schema.txt b/conf/1.0/schema.txt index 52edcc9..8f706d0 100755 --- a/conf/1.0/schema.txt +++ b/conf/1.0/schema.txt @@ -48,4 +48,4 @@ create keyspace TypeConversions with use TypeConversions; create column family UUIDColumnConversion with comparator = TimeUUIDType; create column family SuperUUID with comparator = TimeUUIDType and column_type = Super; - +create column family CompositeColumnConversion with comparator = 'CompositeType(IntegerType, UTF8Type)'; diff --git a/lib/cassandra.rb b/lib/cassandra.rb index 0d6eb1b..f491736 100755 --- a/lib/cassandra.rb +++ b/lib/cassandra.rb @@ -23,6 +23,7 @@ class Cassandra ; end require 'cassandra/time' require 'cassandra/comparable' require 'cassandra/long' +require 'cassandra/composite' require 'cassandra/ordered_hash' require 'cassandra/columns' require 'cassandra/protocol' diff --git a/lib/cassandra/columns.rb b/lib/cassandra/columns.rb index f46ae2d..02db26a 100755 --- a/lib/cassandra/columns.rb +++ b/lib/cassandra/columns.rb @@ -18,10 +18,11 @@ def sub_column_name_class(column_family) def column_name_class_for_key(column_family, comparator_key) property = column_family_property(column_family, comparator_key) - property =~ /.*\.(.*?)$/ + property =~ /[^(]*\.(.*?)$/ case $1 when "LongType" then Long when "LexicalUUIDType", "TimeUUIDType" then SimpleUUID::UUID + when /^CompositeType\(/ then Composite else String # UTF8, Ascii, Bytes, anything else end diff --git a/lib/cassandra/composite.rb b/lib/cassandra/composite.rb new file mode 100644 index 0000000..fabcd05 --- /dev/null +++ b/lib/cassandra/composite.rb @@ -0,0 +1,118 @@ + +class Cassandra + class Composite + include ::Comparable + attr_reader :parts + attr_reader :column_slice + + def initialize(*parts) + options = {} + if parts.last.is_a?(Hash) + options = parts.pop + end + @column_slice = options[:slice] + raise ArgumentError if @column_slice != nil && ![:before, :after].include?(@column_slice) + + if parts.length == 1 && parts[0].instance_of?(self.class) + @column_slice = parts[0].column_slice + @parts = parts[0].parts + elsif parts.length == 1 && parts[0].instance_of?(String) && @column_slice.nil? && valid_packed_composite?(parts[0]) + unpack(parts[0]) + else + @parts = parts + end + end + + def [](*args) + return @parts[*args] + end + + def pack + packed = @parts.map do |part| + [part.length].pack('n') + part + "\x00" + end + if @column_slice + part = @parts[-1] + packed[-1] = [part.length].pack('n') + part + slice_end_of_component + end + return packed.join('') + end + + def to_s + return pack + end + + def <=>(other) + if !other.instance_of?(self.class) + return @parts.first <=> other + end + eoc = slice_end_of_component.unpack('c')[0] + other_eoc = other.slice_end_of_component.unpack('c')[0] + @parts.zip(other.parts).each do |a, b| + next if a == b + if a.nil? && b.nil? + return eoc <=> other_eoc + end + + if a.nil? + return @column_slice == :after ? 1 : -1 + end + if b.nil? + return other.column_slice == :after ? -1 : 1 + end + return -1 if a < b + return 1 if a > b + end + return 0 + end + + def inspect + return "#" + end + + def slice_end_of_component + return "\x01" if @column_slice == :after + return "\xFF" if @column_slice == :before + return "\x00" + end + + private + def unpack(packed_string) + parts = [] + end_of_component = nil + while packed_string.length > 0 + length = packed_string.slice(0, 2).unpack('n')[0] + parts << packed_string.slice(2, length) + end_of_component = packed_string.slice(2 + length, 1) + + packed_string = packed_string.slice(3 + length, packed_string.length) + end + @column_slice = :after if end_of_component == "\x01" + @column_slice = :before if end_of_component == "\xFF" + @parts = parts + end + + def valid_packed_composite?(packed_string) + while packed_string.length > 0 + length = packed_string.slice(0, 2).unpack('n')[0] + return false if length.nil? || length + 3 > packed_string.length + + end_of_component = packed_string.slice(2 + length, 1) + if length + 3 != packed_string.length + return false if end_of_component != "\x00" + end + + packed_string = packed_string.slice(3 + length, packed_string.length) + end + return true + end + + def hash + return to_s.hash + end + + def eql?(other) + return to_s == other.to_s + end + end +end diff --git a/test/cassandra_mock_test.rb b/test/cassandra_mock_test.rb index b3de65a..9397fdb 100755 --- a/test/cassandra_mock_test.rb +++ b/test/cassandra_mock_test.rb @@ -22,6 +22,12 @@ def setup @uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) } @longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) } + @composites = [ + Cassandra::Composite.new([5].pack('N'), "zebra"), + Cassandra::Composite.new([5].pack('N'), "aardvark"), + Cassandra::Composite.new([1].pack('N'), "elephant"), + Cassandra::Composite.new([10].pack('N'), "kangaroo"), + ] end def test_setup diff --git a/test/cassandra_test.rb b/test/cassandra_test.rb index 9e1cfe2..95bc8f9 100755 --- a/test/cassandra_test.rb +++ b/test/cassandra_test.rb @@ -21,6 +21,12 @@ def setup @uuids = (0..6).map {|i| SimpleUUID::UUID.new(Time.at(2**(24+i))) } @longs = (0..6).map {|i| Long.new(Time.at(2**(24+i))) } + @composites = [ + Cassandra::Composite.new([5].pack('N'), "zebra"), + Cassandra::Composite.new([5].pack('N'), "aardvark"), + Cassandra::Composite.new([1].pack('N'), "elephant"), + Cassandra::Composite.new([10].pack('N'), "kangaroo"), + ] end def test_inspect @@ -43,7 +49,7 @@ def test_setting_default_consistency end def test_get_key - + @twitter.insert(:Users, key, {'body' => 'v', 'user' => 'v'}) assert_equal({'body' => 'v', 'user' => 'v'}, @twitter.get(:Users, key)) assert_equal(['body', 'user'].sort, @twitter.get(:Users, key).timestamps.keys.sort) @@ -501,12 +507,12 @@ def test_count_keys def test_count_columns columns = (1..200).inject(Hash.new){|h,v| h['column' + v.to_s] = v.to_s; h;} - + @twitter.insert(:Statuses, key, columns) assert_equal 200, @twitter.count_columns(:Statuses, key, :count => 200) - assert_equal 100, @twitter.count_columns(:Statuses, key) + assert_equal 100, @twitter.count_columns(:Statuses, key) assert_equal 55, @twitter.count_columns(:Statuses, key, :count => 55) - + end def test_count_super_columns @@ -556,24 +562,24 @@ def test_batch_mutate assert_equal({}, @twitter.get(:Users, k + '2')) # Not yet written assert_equal({}, @twitter.get(:Statuses, k + '3')) # Not yet written - @twitter.remove(:Users, k + '1') # Full row + @twitter.remove(:Users, k + '1') # Full row assert_equal({'body' => 'v1', 'user' => 'v1'}, @twitter.get(:Users, k + '1')) # Not yet removed @twitter.remove(:Users, k + '0', 'delete_me') # A single column of the row assert_equal({'delete_me' => 'v0', 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # Not yet removed - + @twitter.remove(:Users, k + '4') @twitter.insert(:Users, k + '4', {'body' => 'v4', 'user' => 'v4'}) assert_equal({}, @twitter.get(:Users, k + '4')) # Not yet written # SuperColumns # Add and delete new sub columns to the user timeline supercolumn - @twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns }) + @twitter.insert(:StatusRelationships, k, {'user_timelines' => new_subcolumns }) @twitter.remove(:StatusRelationships, k, 'user_timelines' , subcolumn_to_delete ) # Delete the first of the initial_subcolumns from the user_timeline supercolumn assert_equal(initial_subcolumns, @twitter.get(:StatusRelationships, k, 'user_timelines')) # No additions or deletes reflected yet - # Delete a complete supercolumn + # Delete a complete supercolumn @twitter.remove(:StatusRelationships, k, 'dummy_supercolumn' ) # Delete the full dummy supercolumn - assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted + assert_equal({@uuids[5] => 'value'}, @twitter.get(:StatusRelationships, k, 'dummy_supercolumn')) # dummy supercolumn not yet deleted end assert_equal({'body' => 'v2', 'user' => 'v2'}, @twitter.get(:Users, k + '2')) # Written @@ -581,9 +587,9 @@ def test_batch_mutate assert_equal({'body' => 'v4', 'user' => 'v4'}, @twitter.get(:Users, k + '4')) # Written assert_equal({'body' => 'v'}, @twitter.get(:Statuses, k + '3')) # Written assert_equal({}, @twitter.get(:Users, k + '1')) # Removed - + assert_equal({ 'keep_me' => 'v0'}, @twitter.get(:Users, k + '0')) # 'delete_me' column removed - + assert_equal({'body' => 'v2', 'user' => 'v2'}.keys.sort, @twitter.get(:Users, k + '2').timestamps.keys.sort) # Written assert_equal({'body' => 'v3', 'user' => 'v3', 'location' => 'v3'}.keys.sort, @twitter.get(:Users, k + '3').timestamps.keys.sort) # Written and compacted @@ -593,7 +599,7 @@ def test_batch_mutate # Final result: initial_subcolumns - initial_subcolumns.first + new_subcolumns resulting_subcolumns = initial_subcolumns.merge(new_subcolumns).reject{|k2,v| k2 == subcolumn_to_delete } assert_equal(resulting_subcolumns, @twitter.get(:StatusRelationships, key, 'user_timelines')) - assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted + assert_equal({}, @twitter.get(:StatusRelationships, key, 'dummy_supercolumn')) # dummy supercolumn deleted end @@ -834,8 +840,47 @@ def test_adding_getting_value_in_multiple_counters_with_super_columns assert_equal(1, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'today')) assert_equal(2, @twitter.get(:UserCounterAggregates, 'bob', 'DAU', 'tomorrow')) end + + def test_composite_column_type_conversion + columns = {} + @composites.each_with_index do |c, index| + columns[c] = "value-#{index}" + end + @type_conversions.insert(:CompositeColumnConversion, key, columns) + columns_in_order = [ + Cassandra::Composite.new([1].pack('N'), "elephant"), + Cassandra::Composite.new([5].pack('N'), "aardvark"), + Cassandra::Composite.new([5].pack('N'), "zebra"), + Cassandra::Composite.new([10].pack('N'), "kangaroo"), + ] + assert_equal(columns_in_order, @type_conversions.get(:CompositeColumnConversion, key).keys) + + column_slice = @type_conversions.get(:CompositeColumnConversion, key, + :start => Cassandra::Composite.new([1].pack('N')), + :finish => Cassandra::Composite.new([10].pack('N')), + ).keys + assert_equal(columns_in_order[0..-2], column_slice) + + column_slice = @type_conversions.get(:CompositeColumnConversion, key, + :start => Cassandra::Composite.new([5].pack('N')), + :finish => Cassandra::Composite.new([5].pack('N'), :slice => :after), + ).keys + assert_equal(columns_in_order[1..2], column_slice) + + column_slice = @type_conversions.get(:CompositeColumnConversion, key, + :start => Cassandra::Composite.new([5].pack('N'), :slice => :after).to_s, + ).keys + assert_equal([columns_in_order[-1]], column_slice) + + column_slice = @type_conversions.get(:CompositeColumnConversion, key, + :finish => Cassandra::Composite.new([10].pack('N'), :slice => :before).to_s, + ).keys + assert_equal(columns_in_order[0..-2], column_slice) + + assert_equal('value-2', @type_conversions.get(:CompositeColumnConversion, key, columns_in_order.first)) + end end - + def test_column_timestamps base_time = Time.now @twitter.insert(:Statuses, "time-key", { "body" => "value" }) diff --git a/test/composite_type_test.rb b/test/composite_type_test.rb new file mode 100644 index 0000000..5280aef --- /dev/null +++ b/test/composite_type_test.rb @@ -0,0 +1,29 @@ +require File.expand_path(File.dirname(__FILE__) + '/test_helper') + +class CompositeTypesTest < Test::Unit::TestCase + include Cassandra::Constants + + def setup + @col_parts = [[363].pack('N'), 'extradites-mulling', SimpleUUID::UUID.new().bytes] + @col = Cassandra::Composite.new(*@col_parts) + end + + def test_creation_from_parts + assert_equal(@col_parts[0], @col[0]) + assert_equal(@col_parts[1], @col[1]) + assert_equal(@col_parts[2], @col[2]) + end + + def test_packing_and_unpacking + part0_length = 2 + 4 + 1 # size + int + end_term + part1_length = 2 + @col_parts[1].length + 1 # size + string_len + end_term + part2_length = 2 + @col_parts[2].length + 1 # size + uuid_bytes + end_term + assert_equal(part0_length + part1_length + part2_length, @col.pack.length) + + col2 = Cassandra::Composite.new(@col.pack) + assert_equal(@col_parts[0], col2[0]) + assert_equal(@col_parts[1], col2[1]) + assert_equal(@col_parts[2], col2[2]) + assert_equal(@col, col2) + end +end